diff --git "a/checkpoint-1690/trainer_state.json" "b/checkpoint-1690/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-1690/trainer_state.json" @@ -0,0 +1,22003 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 1690, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 127.7265625, + "epoch": 0.000591715976331361, + "grad_norm": 0.0, + "kl": 0.0, + "learning_rate": 9.994082840236685e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1 + }, + { + "completion_length": 110.6640625, + "epoch": 0.001183431952662722, + "grad_norm": 0.34203776424753146, + "kl": 0.0, + "learning_rate": 9.988165680473373e-07, + "loss": 0.0, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 2 + }, + { + "completion_length": 131.796875, + "epoch": 0.0017751479289940828, + "grad_norm": 0.30532368803795273, + "kl": 0.000484466552734375, + "learning_rate": 9.98224852071006e-07, + "loss": 0.0, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 3 + }, + { + "completion_length": 120.7265625, + "epoch": 0.002366863905325444, + "grad_norm": 0.4257266173178317, + "kl": 0.000499725341796875, + "learning_rate": 9.976331360946745e-07, + "loss": 0.0, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 4 + }, + { + "completion_length": 127.875, + "epoch": 0.0029585798816568047, + "grad_norm": 0.6013481878720278, + "kl": 0.000701904296875, + "learning_rate": 9.97041420118343e-07, + "loss": 0.0, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 5 + }, + { + "completion_length": 126.2734375, + "epoch": 0.0035502958579881655, + "grad_norm": 0.3252081857250971, + "kl": 0.00057220458984375, + "learning_rate": 9.964497041420118e-07, + "loss": 0.0, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 6 + }, + { + "completion_length": 123.453125, + "epoch": 0.004142011834319527, + "grad_norm": 0.4912733233639219, + "kl": 0.000732421875, + "learning_rate": 9.958579881656804e-07, + "loss": 0.0, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 7 + }, + { + "completion_length": 114.078125, + "epoch": 0.004733727810650888, + "grad_norm": 0.3167135184248363, + "kl": 0.00064849853515625, + "learning_rate": 9.952662721893492e-07, + "loss": 0.0, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 8 + }, + { + "completion_length": 126.453125, + "epoch": 0.0053254437869822485, + "grad_norm": 0.01906110397026569, + "kl": 0.000621795654296875, + "learning_rate": 9.946745562130176e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 9 + }, + { + "completion_length": 126.0390625, + "epoch": 0.005917159763313609, + "grad_norm": 0.47572494531607395, + "kl": 0.000701904296875, + "learning_rate": 9.940828402366864e-07, + "loss": 0.0, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 10 + }, + { + "completion_length": 125.875, + "epoch": 0.00650887573964497, + "grad_norm": 0.4655948860801345, + "kl": 0.0010223388671875, + "learning_rate": 9.93491124260355e-07, + "loss": 0.0, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 11 + }, + { + "completion_length": 117.6953125, + "epoch": 0.007100591715976331, + "grad_norm": 0.36159963989174815, + "kl": 0.000835418701171875, + "learning_rate": 9.928994082840237e-07, + "loss": 0.0, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 12 + }, + { + "completion_length": 132.484375, + "epoch": 0.007692307692307693, + "grad_norm": 0.43477392303656953, + "kl": 0.00099945068359375, + "learning_rate": 9.923076923076923e-07, + "loss": 0.0, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 13 + }, + { + "completion_length": 123.6171875, + "epoch": 0.008284023668639054, + "grad_norm": 0.04902113685812249, + "kl": 0.00145721435546875, + "learning_rate": 9.917159763313609e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 14 + }, + { + "completion_length": 125.4140625, + "epoch": 0.008875739644970414, + "grad_norm": 0.583464211795418, + "kl": 0.004364013671875, + "learning_rate": 9.911242603550295e-07, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 15 + }, + { + "completion_length": 121.3828125, + "epoch": 0.009467455621301775, + "grad_norm": 0.46040483586105996, + "kl": 0.00162506103515625, + "learning_rate": 9.905325443786982e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 16 + }, + { + "completion_length": 113.7734375, + "epoch": 0.010059171597633136, + "grad_norm": 0.6151321909201266, + "kl": 0.00140380859375, + "learning_rate": 9.899408284023668e-07, + "loss": 0.0001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 17 + }, + { + "completion_length": 119.9765625, + "epoch": 0.010650887573964497, + "grad_norm": 0.0288947402127287, + "kl": 0.001220703125, + "learning_rate": 9.893491124260354e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 18 + }, + { + "completion_length": 115.78125, + "epoch": 0.011242603550295858, + "grad_norm": 0.4168234213497549, + "kl": 0.001434326171875, + "learning_rate": 9.887573964497042e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 19 + }, + { + "completion_length": 124.03125, + "epoch": 0.011834319526627219, + "grad_norm": 0.6990714963851965, + "kl": 0.0013580322265625, + "learning_rate": 9.881656804733728e-07, + "loss": 0.0001, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 20 + }, + { + "completion_length": 123.9921875, + "epoch": 0.01242603550295858, + "grad_norm": 2.0841325445313754, + "kl": 0.0015716552734375, + "learning_rate": 9.875739644970413e-07, + "loss": 0.0001, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 21 + }, + { + "completion_length": 129.8984375, + "epoch": 0.01301775147928994, + "grad_norm": 0.8477754944913156, + "kl": 0.00147247314453125, + "learning_rate": 9.8698224852071e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 22 + }, + { + "completion_length": 128.1015625, + "epoch": 0.013609467455621301, + "grad_norm": 0.38969365239151915, + "kl": 0.0015716552734375, + "learning_rate": 9.863905325443787e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 23 + }, + { + "completion_length": 129.1875, + "epoch": 0.014201183431952662, + "grad_norm": 0.8190645814711185, + "kl": 0.00182342529296875, + "learning_rate": 9.857988165680473e-07, + "loss": 0.0001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 24 + }, + { + "completion_length": 107.8359375, + "epoch": 0.014792899408284023, + "grad_norm": 0.6278110561815192, + "kl": 0.00189208984375, + "learning_rate": 9.852071005917159e-07, + "loss": 0.0001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 25 + }, + { + "completion_length": 121.65625, + "epoch": 0.015384615384615385, + "grad_norm": 0.2962842610296049, + "kl": 0.002288818359375, + "learning_rate": 9.846153846153847e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 26 + }, + { + "completion_length": 128.7265625, + "epoch": 0.015976331360946745, + "grad_norm": 0.4631999776784921, + "kl": 0.0025634765625, + "learning_rate": 9.840236686390532e-07, + "loss": 0.0001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 27 + }, + { + "completion_length": 119.5234375, + "epoch": 0.016568047337278107, + "grad_norm": 0.04692510874272438, + "kl": 0.002410888671875, + "learning_rate": 9.834319526627218e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 28 + }, + { + "completion_length": 112.5078125, + "epoch": 0.017159763313609466, + "grad_norm": 0.049000181497653415, + "kl": 0.002532958984375, + "learning_rate": 9.828402366863904e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 29 + }, + { + "completion_length": 113.3359375, + "epoch": 0.01775147928994083, + "grad_norm": 1.5047038307867133, + "kl": 0.0026092529296875, + "learning_rate": 9.822485207100592e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 30 + }, + { + "completion_length": 118.8515625, + "epoch": 0.018343195266272188, + "grad_norm": 0.04870420805876279, + "kl": 0.002685546875, + "learning_rate": 9.816568047337277e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 31 + }, + { + "completion_length": 115.71875, + "epoch": 0.01893491124260355, + "grad_norm": 0.5883754202863636, + "kl": 0.0027923583984375, + "learning_rate": 9.810650887573963e-07, + "loss": 0.0001, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 32 + }, + { + "completion_length": 120.328125, + "epoch": 0.01952662721893491, + "grad_norm": 0.3117407439153625, + "kl": 0.00274658203125, + "learning_rate": 9.804733727810651e-07, + "loss": 0.0001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 33 + }, + { + "completion_length": 124.625, + "epoch": 0.020118343195266272, + "grad_norm": 0.2409059284290115, + "kl": 0.0023956298828125, + "learning_rate": 9.798816568047337e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 34 + }, + { + "completion_length": 120.8125, + "epoch": 0.020710059171597635, + "grad_norm": 0.7345060130723324, + "kl": 0.003265380859375, + "learning_rate": 9.792899408284023e-07, + "loss": 0.0001, + "reward": 0.0625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 35 + }, + { + "completion_length": 120.6171875, + "epoch": 0.021301775147928994, + "grad_norm": 0.3414521559185693, + "kl": 0.0030364990234375, + "learning_rate": 9.78698224852071e-07, + "loss": 0.0001, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 36 + }, + { + "completion_length": 123.03125, + "epoch": 0.021893491124260357, + "grad_norm": 0.29701099206860027, + "kl": 0.003204345703125, + "learning_rate": 9.781065088757396e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 37 + }, + { + "completion_length": 121.9296875, + "epoch": 0.022485207100591716, + "grad_norm": 0.03287035155878542, + "kl": 0.00262451171875, + "learning_rate": 9.775147928994082e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 38 + }, + { + "completion_length": 126.59375, + "epoch": 0.023076923076923078, + "grad_norm": 0.31422862751800684, + "kl": 0.00384521484375, + "learning_rate": 9.769230769230768e-07, + "loss": 0.0002, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 39 + }, + { + "completion_length": 134.078125, + "epoch": 0.023668639053254437, + "grad_norm": 0.4165883578187062, + "kl": 0.00372314453125, + "learning_rate": 9.763313609467456e-07, + "loss": 0.0001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.015625, + "step": 40 + }, + { + "completion_length": 133.1875, + "epoch": 0.0242603550295858, + "grad_norm": 0.5443197038086571, + "kl": 0.0035400390625, + "learning_rate": 9.757396449704142e-07, + "loss": 0.0001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 41 + }, + { + "completion_length": 123.0078125, + "epoch": 0.02485207100591716, + "grad_norm": 0.5514891399452323, + "kl": 0.00439453125, + "learning_rate": 9.751479289940827e-07, + "loss": 0.0002, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 42 + }, + { + "completion_length": 115.4921875, + "epoch": 0.02544378698224852, + "grad_norm": 0.8324525814256356, + "kl": 0.003570556640625, + "learning_rate": 9.745562130177515e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 43 + }, + { + "completion_length": 115.578125, + "epoch": 0.02603550295857988, + "grad_norm": 0.3736217073595516, + "kl": 0.00360107421875, + "learning_rate": 9.7396449704142e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 44 + }, + { + "completion_length": 118.8828125, + "epoch": 0.026627218934911243, + "grad_norm": 0.6194848366608472, + "kl": 0.0052490234375, + "learning_rate": 9.733727810650887e-07, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 45 + }, + { + "completion_length": 123.203125, + "epoch": 0.027218934911242602, + "grad_norm": 0.4049431753294096, + "kl": 0.002960205078125, + "learning_rate": 9.727810650887573e-07, + "loss": 0.0001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 46 + }, + { + "completion_length": 121.8984375, + "epoch": 0.027810650887573965, + "grad_norm": 0.4590313706434288, + "kl": 0.004119873046875, + "learning_rate": 9.72189349112426e-07, + "loss": 0.0002, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 47 + }, + { + "completion_length": 125.5625, + "epoch": 0.028402366863905324, + "grad_norm": 1.5581842728072088, + "kl": 0.0034942626953125, + "learning_rate": 9.715976331360946e-07, + "loss": 0.0001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 48 + }, + { + "completion_length": 124.03125, + "epoch": 0.028994082840236687, + "grad_norm": 0.8021799922553275, + "kl": 0.0034637451171875, + "learning_rate": 9.710059171597634e-07, + "loss": 0.0001, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 49 + }, + { + "completion_length": 123.4453125, + "epoch": 0.029585798816568046, + "grad_norm": 0.29777857337611535, + "kl": 0.00323486328125, + "learning_rate": 9.704142011834318e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 50 + }, + { + "completion_length": 122.3125, + "epoch": 0.03017751479289941, + "grad_norm": 0.5631714211262429, + "kl": 0.0037841796875, + "learning_rate": 9.698224852071006e-07, + "loss": 0.0002, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 51 + }, + { + "completion_length": 127.1171875, + "epoch": 0.03076923076923077, + "grad_norm": 0.4283958064875684, + "kl": 0.003936767578125, + "learning_rate": 9.692307692307691e-07, + "loss": 0.0002, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 52 + }, + { + "completion_length": 127.9296875, + "epoch": 0.031360946745562134, + "grad_norm": 0.28762692674947626, + "kl": 0.0032958984375, + "learning_rate": 9.68639053254438e-07, + "loss": 0.0001, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 53 + }, + { + "completion_length": 119.65625, + "epoch": 0.03195266272189349, + "grad_norm": 0.7180552394781214, + "kl": 0.004547119140625, + "learning_rate": 9.680473372781065e-07, + "loss": 0.0002, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 54 + }, + { + "completion_length": 115.5390625, + "epoch": 0.03254437869822485, + "grad_norm": 0.7654149001611007, + "kl": 0.0048828125, + "learning_rate": 9.67455621301775e-07, + "loss": 0.0002, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 55 + }, + { + "completion_length": 122.3046875, + "epoch": 0.033136094674556214, + "grad_norm": 0.6189662170851733, + "kl": 0.004547119140625, + "learning_rate": 9.668639053254437e-07, + "loss": 0.0002, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 56 + }, + { + "completion_length": 115.40625, + "epoch": 0.03372781065088758, + "grad_norm": 0.457308707922435, + "kl": 0.00506591796875, + "learning_rate": 9.662721893491124e-07, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 57 + }, + { + "completion_length": 126.6796875, + "epoch": 0.03431952662721893, + "grad_norm": 0.4589917072521562, + "kl": 0.00469970703125, + "learning_rate": 9.65680473372781e-07, + "loss": 0.0002, + "reward": 0.0234375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.015625, + "step": 58 + }, + { + "completion_length": 122.8828125, + "epoch": 0.034911242603550295, + "grad_norm": 0.6843236879362687, + "kl": 0.00457763671875, + "learning_rate": 9.650887573964496e-07, + "loss": 0.0002, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 59 + }, + { + "completion_length": 126.625, + "epoch": 0.03550295857988166, + "grad_norm": 0.06202256749496351, + "kl": 0.00421142578125, + "learning_rate": 9.644970414201184e-07, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 60 + }, + { + "completion_length": 133.5703125, + "epoch": 0.03609467455621302, + "grad_norm": 0.41007435668864184, + "kl": 0.00390625, + "learning_rate": 9.63905325443787e-07, + "loss": 0.0002, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 61 + }, + { + "completion_length": 116.5625, + "epoch": 0.036686390532544376, + "grad_norm": 0.36393661387252874, + "kl": 0.004180908203125, + "learning_rate": 9.633136094674555e-07, + "loss": 0.0002, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 62 + }, + { + "completion_length": 122.3203125, + "epoch": 0.03727810650887574, + "grad_norm": 0.06027245502191043, + "kl": 0.005401611328125, + "learning_rate": 9.627218934911241e-07, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 63 + }, + { + "completion_length": 134.125, + "epoch": 0.0378698224852071, + "grad_norm": 0.10085659582496373, + "kl": 0.00823974609375, + "learning_rate": 9.62130177514793e-07, + "loss": 0.0003, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 64 + }, + { + "completion_length": 125.4609375, + "epoch": 0.038461538461538464, + "grad_norm": 0.5865375268727395, + "kl": 0.006622314453125, + "learning_rate": 9.615384615384615e-07, + "loss": 0.0003, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 65 + }, + { + "completion_length": 130.46875, + "epoch": 0.03905325443786982, + "grad_norm": 0.05901727626444257, + "kl": 0.005035400390625, + "learning_rate": 9.609467455621303e-07, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 66 + }, + { + "completion_length": 119.7265625, + "epoch": 0.03964497041420118, + "grad_norm": 0.47742397488805327, + "kl": 0.005096435546875, + "learning_rate": 9.603550295857986e-07, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 67 + }, + { + "completion_length": 123.4140625, + "epoch": 0.040236686390532544, + "grad_norm": 0.2565945886022302, + "kl": 0.005035400390625, + "learning_rate": 9.597633136094674e-07, + "loss": 0.0002, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 68 + }, + { + "completion_length": 125.53125, + "epoch": 0.04082840236686391, + "grad_norm": 0.4894682558839944, + "kl": 0.0072021484375, + "learning_rate": 9.59171597633136e-07, + "loss": 0.0003, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 69 + }, + { + "completion_length": 127.6328125, + "epoch": 0.04142011834319527, + "grad_norm": 0.5637599802922832, + "kl": 0.007476806640625, + "learning_rate": 9.585798816568048e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 70 + }, + { + "completion_length": 125.3671875, + "epoch": 0.042011834319526625, + "grad_norm": 1.9476952563456562, + "kl": 0.00732421875, + "learning_rate": 9.579881656804734e-07, + "loss": 0.0003, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 71 + }, + { + "completion_length": 125.7890625, + "epoch": 0.04260355029585799, + "grad_norm": 0.5500757420747228, + "kl": 0.008544921875, + "learning_rate": 9.57396449704142e-07, + "loss": 0.0003, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 72 + }, + { + "completion_length": 132.2890625, + "epoch": 0.04319526627218935, + "grad_norm": 0.06906506040365526, + "kl": 0.00555419921875, + "learning_rate": 9.568047337278105e-07, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 73 + }, + { + "completion_length": 128.1171875, + "epoch": 0.04378698224852071, + "grad_norm": 0.4103641284733186, + "kl": 0.005889892578125, + "learning_rate": 9.562130177514793e-07, + "loss": 0.0002, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 74 + }, + { + "completion_length": 124.734375, + "epoch": 0.04437869822485207, + "grad_norm": 0.5524943051276718, + "kl": 0.0069580078125, + "learning_rate": 9.556213017751479e-07, + "loss": 0.0003, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 75 + }, + { + "completion_length": 131.1484375, + "epoch": 0.04497041420118343, + "grad_norm": 0.43545385218874555, + "kl": 0.006622314453125, + "learning_rate": 9.550295857988165e-07, + "loss": 0.0003, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 76 + }, + { + "completion_length": 127.0703125, + "epoch": 0.045562130177514794, + "grad_norm": 0.5451691321741313, + "kl": 0.006622314453125, + "learning_rate": 9.544378698224853e-07, + "loss": 0.0003, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 77 + }, + { + "completion_length": 122.375, + "epoch": 0.046153846153846156, + "grad_norm": 0.2899457483806303, + "kl": 0.00677490234375, + "learning_rate": 9.538461538461538e-07, + "loss": 0.0003, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 78 + }, + { + "completion_length": 127.234375, + "epoch": 0.04674556213017751, + "grad_norm": 0.8396853655445932, + "kl": 0.006744384765625, + "learning_rate": 9.532544378698224e-07, + "loss": 0.0003, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 79 + }, + { + "completion_length": 136.1796875, + "epoch": 0.047337278106508875, + "grad_norm": 0.7005016462392128, + "kl": 0.006378173828125, + "learning_rate": 9.526627218934911e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 80 + }, + { + "completion_length": 135.296875, + "epoch": 0.04792899408284024, + "grad_norm": 0.23593983757462866, + "kl": 0.0050048828125, + "learning_rate": 9.520710059171597e-07, + "loss": 0.0002, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 81 + }, + { + "completion_length": 121.984375, + "epoch": 0.0485207100591716, + "grad_norm": 0.5694508747059174, + "kl": 0.0089111328125, + "learning_rate": 9.514792899408284e-07, + "loss": 0.0004, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 82 + }, + { + "completion_length": 133.6484375, + "epoch": 0.049112426035502955, + "grad_norm": 0.551072738711201, + "kl": 0.00640869140625, + "learning_rate": 9.50887573964497e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 83 + }, + { + "completion_length": 121.5859375, + "epoch": 0.04970414201183432, + "grad_norm": 0.3088442940087554, + "kl": 0.007354736328125, + "learning_rate": 9.502958579881656e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 84 + }, + { + "completion_length": 122.8671875, + "epoch": 0.05029585798816568, + "grad_norm": 0.7736527276306868, + "kl": 0.008056640625, + "learning_rate": 9.497041420118342e-07, + "loss": 0.0003, + "reward": 0.078125, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 85 + }, + { + "completion_length": 122.171875, + "epoch": 0.05088757396449704, + "grad_norm": 0.47052733792762674, + "kl": 0.00616455078125, + "learning_rate": 9.491124260355029e-07, + "loss": 0.0002, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 86 + }, + { + "completion_length": 124.6640625, + "epoch": 0.051479289940828406, + "grad_norm": 0.31335420768416905, + "kl": 0.008056640625, + "learning_rate": 9.485207100591716e-07, + "loss": 0.0003, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 87 + }, + { + "completion_length": 131.1171875, + "epoch": 0.05207100591715976, + "grad_norm": 0.4135595653189128, + "kl": 0.006622314453125, + "learning_rate": 9.479289940828402e-07, + "loss": 0.0003, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 88 + }, + { + "completion_length": 117.7734375, + "epoch": 0.052662721893491124, + "grad_norm": 0.37729321699714397, + "kl": 0.007598876953125, + "learning_rate": 9.473372781065089e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 89 + }, + { + "completion_length": 115.6328125, + "epoch": 0.05325443786982249, + "grad_norm": 0.35623437167566996, + "kl": 0.006072998046875, + "learning_rate": 9.467455621301774e-07, + "loss": 0.0002, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 90 + }, + { + "completion_length": 122.15625, + "epoch": 0.05384615384615385, + "grad_norm": 0.056199919689822804, + "kl": 0.007110595703125, + "learning_rate": 9.461538461538461e-07, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 91 + }, + { + "completion_length": 116.09375, + "epoch": 0.054437869822485205, + "grad_norm": 0.7241169655484281, + "kl": 0.0108642578125, + "learning_rate": 9.455621301775148e-07, + "loss": 0.0004, + "reward": 0.1015625, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 92 + }, + { + "completion_length": 117.421875, + "epoch": 0.05502958579881657, + "grad_norm": 0.5790248004082525, + "kl": 0.00811767578125, + "learning_rate": 9.449704142011834e-07, + "loss": 0.0003, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 93 + }, + { + "completion_length": 110.609375, + "epoch": 0.05562130177514793, + "grad_norm": 0.5421011073909128, + "kl": 0.00689697265625, + "learning_rate": 9.44378698224852e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 94 + }, + { + "completion_length": 120.921875, + "epoch": 0.05621301775147929, + "grad_norm": 0.0791177830677674, + "kl": 0.00970458984375, + "learning_rate": 9.437869822485207e-07, + "loss": 0.0004, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 95 + }, + { + "completion_length": 124.890625, + "epoch": 0.05680473372781065, + "grad_norm": 0.3585406922133411, + "kl": 0.00933837890625, + "learning_rate": 9.431952662721893e-07, + "loss": 0.0004, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 96 + }, + { + "completion_length": 124.734375, + "epoch": 0.05739644970414201, + "grad_norm": 0.45709308266379617, + "kl": 0.00830078125, + "learning_rate": 9.42603550295858e-07, + "loss": 0.0003, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 97 + }, + { + "completion_length": 123.765625, + "epoch": 0.05798816568047337, + "grad_norm": 0.3996745582694027, + "kl": 0.0079345703125, + "learning_rate": 9.420118343195265e-07, + "loss": 0.0003, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 98 + }, + { + "completion_length": 126.3828125, + "epoch": 0.058579881656804736, + "grad_norm": 0.46031969972240927, + "kl": 0.00762939453125, + "learning_rate": 9.414201183431952e-07, + "loss": 0.0003, + "reward": 0.0234375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 99 + }, + { + "completion_length": 119.3046875, + "epoch": 0.05917159763313609, + "grad_norm": 1.8394680198420486, + "kl": 0.01324462890625, + "learning_rate": 9.408284023668639e-07, + "loss": 0.0005, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 100 + }, + { + "completion_length": 115.7109375, + "epoch": 0.059763313609467454, + "grad_norm": 0.4611888455188371, + "kl": 0.0093994140625, + "learning_rate": 9.402366863905326e-07, + "loss": 0.0004, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 101 + }, + { + "completion_length": 117.0234375, + "epoch": 0.06035502958579882, + "grad_norm": 0.2979772700495936, + "kl": 0.0081787109375, + "learning_rate": 9.396449704142011e-07, + "loss": 0.0003, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 102 + }, + { + "completion_length": 123.2265625, + "epoch": 0.06094674556213018, + "grad_norm": 0.41320177025757404, + "kl": 0.0062255859375, + "learning_rate": 9.390532544378697e-07, + "loss": 0.0002, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 103 + }, + { + "completion_length": 110.6328125, + "epoch": 0.06153846153846154, + "grad_norm": 0.4582520213915961, + "kl": 0.00860595703125, + "learning_rate": 9.384615384615384e-07, + "loss": 0.0003, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 104 + }, + { + "completion_length": 121.3203125, + "epoch": 0.0621301775147929, + "grad_norm": 0.05420322780022324, + "kl": 0.00799560546875, + "learning_rate": 9.378698224852071e-07, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 105 + }, + { + "completion_length": 123.484375, + "epoch": 0.06272189349112427, + "grad_norm": 0.26461294237990546, + "kl": 0.0093994140625, + "learning_rate": 9.372781065088757e-07, + "loss": 0.0004, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 106 + }, + { + "completion_length": 121.59375, + "epoch": 0.06331360946745562, + "grad_norm": 0.421403573940906, + "kl": 0.0113525390625, + "learning_rate": 9.366863905325444e-07, + "loss": 0.0005, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 107 + }, + { + "completion_length": 136.3828125, + "epoch": 0.06390532544378698, + "grad_norm": 0.6537863786775717, + "kl": 0.0125732421875, + "learning_rate": 9.36094674556213e-07, + "loss": 0.0005, + "reward": 0.0625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 108 + }, + { + "completion_length": 112.1328125, + "epoch": 0.06449704142011835, + "grad_norm": 0.36636230807325765, + "kl": 0.00897216796875, + "learning_rate": 9.355029585798816e-07, + "loss": 0.0004, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 109 + }, + { + "completion_length": 124.5, + "epoch": 0.0650887573964497, + "grad_norm": 0.5817523250809808, + "kl": 0.010986328125, + "learning_rate": 9.349112426035502e-07, + "loss": 0.0004, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 110 + }, + { + "completion_length": 120.65625, + "epoch": 0.06568047337278106, + "grad_norm": 0.6549214480495309, + "kl": 0.0135498046875, + "learning_rate": 9.343195266272189e-07, + "loss": 0.0005, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 111 + }, + { + "completion_length": 127.7578125, + "epoch": 0.06627218934911243, + "grad_norm": 0.7214273934610367, + "kl": 0.01019287109375, + "learning_rate": 9.337278106508876e-07, + "loss": 0.0004, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 112 + }, + { + "completion_length": 111.9609375, + "epoch": 0.06686390532544378, + "grad_norm": 0.39479882034977537, + "kl": 0.01123046875, + "learning_rate": 9.331360946745562e-07, + "loss": 0.0004, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 113 + }, + { + "completion_length": 116.4375, + "epoch": 0.06745562130177515, + "grad_norm": 0.5975471171678186, + "kl": 0.01385498046875, + "learning_rate": 9.325443786982247e-07, + "loss": 0.0006, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 114 + }, + { + "completion_length": 113.171875, + "epoch": 0.06804733727810651, + "grad_norm": 0.5554674049544309, + "kl": 0.01263427734375, + "learning_rate": 9.319526627218934e-07, + "loss": 0.0005, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 115 + }, + { + "completion_length": 124.84375, + "epoch": 0.06863905325443787, + "grad_norm": 0.7387544789432274, + "kl": 0.0150146484375, + "learning_rate": 9.313609467455621e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 116 + }, + { + "completion_length": 117.9296875, + "epoch": 0.06923076923076923, + "grad_norm": 0.591352989300411, + "kl": 0.01171875, + "learning_rate": 9.307692307692308e-07, + "loss": 0.0005, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 117 + }, + { + "completion_length": 113.328125, + "epoch": 0.06982248520710059, + "grad_norm": 0.8605094942388973, + "kl": 0.011962890625, + "learning_rate": 9.301775147928995e-07, + "loss": 0.0005, + "reward": 0.109375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 118 + }, + { + "completion_length": 129.5859375, + "epoch": 0.07041420118343195, + "grad_norm": 0.5015625837899907, + "kl": 0.01318359375, + "learning_rate": 9.295857988165679e-07, + "loss": 0.0005, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 119 + }, + { + "completion_length": 113.8984375, + "epoch": 0.07100591715976332, + "grad_norm": 0.4500020327792315, + "kl": 0.012939453125, + "learning_rate": 9.289940828402366e-07, + "loss": 0.0005, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 120 + }, + { + "completion_length": 110.828125, + "epoch": 0.07159763313609467, + "grad_norm": 0.39556007686477235, + "kl": 0.0120849609375, + "learning_rate": 9.284023668639053e-07, + "loss": 0.0005, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 121 + }, + { + "completion_length": 117.359375, + "epoch": 0.07218934911242604, + "grad_norm": 0.938433620915168, + "kl": 0.0128173828125, + "learning_rate": 9.27810650887574e-07, + "loss": 0.0005, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 122 + }, + { + "completion_length": 114.1015625, + "epoch": 0.0727810650887574, + "grad_norm": 0.43034687432859886, + "kl": 0.0123291015625, + "learning_rate": 9.272189349112426e-07, + "loss": 0.0005, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 123 + }, + { + "completion_length": 107.8671875, + "epoch": 0.07337278106508875, + "grad_norm": 1.0135628770285412, + "kl": 0.01495361328125, + "learning_rate": 9.266272189349112e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 124 + }, + { + "completion_length": 119.96875, + "epoch": 0.07396449704142012, + "grad_norm": 0.6414541364185019, + "kl": 0.01025390625, + "learning_rate": 9.260355029585798e-07, + "loss": 0.0004, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 125 + }, + { + "completion_length": 119.671875, + "epoch": 0.07455621301775148, + "grad_norm": 0.6904477718014338, + "kl": 0.01312255859375, + "learning_rate": 9.254437869822485e-07, + "loss": 0.0005, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 126 + }, + { + "completion_length": 122.6953125, + "epoch": 0.07514792899408285, + "grad_norm": 0.5910067318862893, + "kl": 0.01611328125, + "learning_rate": 9.248520710059171e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 127 + }, + { + "completion_length": 128.8828125, + "epoch": 0.0757396449704142, + "grad_norm": 0.31467782958162116, + "kl": 0.0111083984375, + "learning_rate": 9.242603550295858e-07, + "loss": 0.0004, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 128 + }, + { + "completion_length": 117.546875, + "epoch": 0.07633136094674556, + "grad_norm": 0.670697943448764, + "kl": 0.0137939453125, + "learning_rate": 9.236686390532544e-07, + "loss": 0.0006, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 129 + }, + { + "completion_length": 113.3359375, + "epoch": 0.07692307692307693, + "grad_norm": 0.5655672576989389, + "kl": 0.01251220703125, + "learning_rate": 9.230769230769231e-07, + "loss": 0.0005, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 130 + }, + { + "completion_length": 114.5703125, + "epoch": 0.07751479289940828, + "grad_norm": 0.6770817717181619, + "kl": 0.01251220703125, + "learning_rate": 9.224852071005916e-07, + "loss": 0.0005, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 131 + }, + { + "completion_length": 119.03125, + "epoch": 0.07810650887573964, + "grad_norm": 0.05845909955913872, + "kl": 0.010498046875, + "learning_rate": 9.218934911242603e-07, + "loss": 0.0004, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 132 + }, + { + "completion_length": 120.953125, + "epoch": 0.07869822485207101, + "grad_norm": 0.5939419934528988, + "kl": 0.015380859375, + "learning_rate": 9.21301775147929e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 133 + }, + { + "completion_length": 113.0546875, + "epoch": 0.07928994082840236, + "grad_norm": 0.9014864940398385, + "kl": 0.01409912109375, + "learning_rate": 9.207100591715976e-07, + "loss": 0.0006, + "reward": 0.1171875, + "reward_std": 0.11420939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 134 + }, + { + "completion_length": 115.3671875, + "epoch": 0.07988165680473373, + "grad_norm": 0.4015822146785734, + "kl": 0.01385498046875, + "learning_rate": 9.201183431952662e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 135 + }, + { + "completion_length": 127.7890625, + "epoch": 0.08047337278106509, + "grad_norm": 0.533719644644771, + "kl": 0.0169677734375, + "learning_rate": 9.195266272189349e-07, + "loss": 0.0007, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 136 + }, + { + "completion_length": 122.5625, + "epoch": 0.08106508875739644, + "grad_norm": 0.5344322227208266, + "kl": 0.0137939453125, + "learning_rate": 9.189349112426035e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 137 + }, + { + "completion_length": 128.9453125, + "epoch": 0.08165680473372781, + "grad_norm": 0.7968399108309859, + "kl": 0.01495361328125, + "learning_rate": 9.183431952662722e-07, + "loss": 0.0006, + "reward": 0.0703125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 138 + }, + { + "completion_length": 127.7265625, + "epoch": 0.08224852071005917, + "grad_norm": 0.38488622160175245, + "kl": 0.0137939453125, + "learning_rate": 9.177514792899407e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 139 + }, + { + "completion_length": 118.984375, + "epoch": 0.08284023668639054, + "grad_norm": 0.8246014323648274, + "kl": 0.01470947265625, + "learning_rate": 9.171597633136094e-07, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 140 + }, + { + "completion_length": 126.8359375, + "epoch": 0.0834319526627219, + "grad_norm": 0.6961287288054611, + "kl": 0.013671875, + "learning_rate": 9.165680473372781e-07, + "loss": 0.0005, + "reward": 0.0546875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 141 + }, + { + "completion_length": 129.1171875, + "epoch": 0.08402366863905325, + "grad_norm": 0.544735023191767, + "kl": 0.01507568359375, + "learning_rate": 9.159763313609467e-07, + "loss": 0.0006, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 142 + }, + { + "completion_length": 131.296875, + "epoch": 0.08461538461538462, + "grad_norm": 0.34062156433097973, + "kl": 0.0140380859375, + "learning_rate": 9.153846153846153e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 143 + }, + { + "completion_length": 127.4296875, + "epoch": 0.08520710059171598, + "grad_norm": 0.6941615536119394, + "kl": 0.0152587890625, + "learning_rate": 9.147928994082839e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 144 + }, + { + "completion_length": 126.21875, + "epoch": 0.08579881656804733, + "grad_norm": 0.8946481861173489, + "kl": 0.0155029296875, + "learning_rate": 9.142011834319526e-07, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 145 + }, + { + "completion_length": 124.6640625, + "epoch": 0.0863905325443787, + "grad_norm": 0.4809501623968118, + "kl": 0.0157470703125, + "learning_rate": 9.136094674556213e-07, + "loss": 0.0006, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 146 + }, + { + "completion_length": 128.6328125, + "epoch": 0.08698224852071006, + "grad_norm": 0.057576430474296295, + "kl": 0.01123046875, + "learning_rate": 9.1301775147929e-07, + "loss": 0.0004, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 147 + }, + { + "completion_length": 122.4375, + "epoch": 0.08757396449704143, + "grad_norm": 0.5340406324192233, + "kl": 0.01507568359375, + "learning_rate": 9.124260355029585e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 148 + }, + { + "completion_length": 123.828125, + "epoch": 0.08816568047337278, + "grad_norm": 1.1302527487641498, + "kl": 0.017822265625, + "learning_rate": 9.118343195266271e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 149 + }, + { + "completion_length": 124.703125, + "epoch": 0.08875739644970414, + "grad_norm": 0.7231313914815516, + "kl": 0.019287109375, + "learning_rate": 9.112426035502958e-07, + "loss": 0.0008, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 150 + }, + { + "completion_length": 126.3671875, + "epoch": 0.0893491124260355, + "grad_norm": 0.2626987937937861, + "kl": 0.0123291015625, + "learning_rate": 9.106508875739645e-07, + "loss": 0.0005, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 151 + }, + { + "completion_length": 126.625, + "epoch": 0.08994082840236686, + "grad_norm": 0.3259993960363598, + "kl": 0.0223388671875, + "learning_rate": 9.100591715976331e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 152 + }, + { + "completion_length": 119.2578125, + "epoch": 0.09053254437869822, + "grad_norm": 0.4614198785736164, + "kl": 0.02099609375, + "learning_rate": 9.094674556213018e-07, + "loss": 0.0008, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 153 + }, + { + "completion_length": 123.46875, + "epoch": 0.09112426035502959, + "grad_norm": 0.4300188576668633, + "kl": 0.0167236328125, + "learning_rate": 9.088757396449704e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 154 + }, + { + "completion_length": 120.859375, + "epoch": 0.09171597633136094, + "grad_norm": 0.49577278648322054, + "kl": 0.024658203125, + "learning_rate": 9.08284023668639e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 155 + }, + { + "completion_length": 117.9375, + "epoch": 0.09230769230769231, + "grad_norm": 0.6962492600889447, + "kl": 0.0267333984375, + "learning_rate": 9.076923076923076e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 156 + }, + { + "completion_length": 130.65625, + "epoch": 0.09289940828402367, + "grad_norm": 2.100537265955606, + "kl": 0.0184326171875, + "learning_rate": 9.071005917159763e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 157 + }, + { + "completion_length": 113.3125, + "epoch": 0.09349112426035502, + "grad_norm": 0.364326410311684, + "kl": 0.0177001953125, + "learning_rate": 9.06508875739645e-07, + "loss": 0.0007, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 158 + }, + { + "completion_length": 126.6640625, + "epoch": 0.0940828402366864, + "grad_norm": 0.8858316074827334, + "kl": 0.0216064453125, + "learning_rate": 9.059171597633137e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 159 + }, + { + "completion_length": 131.78125, + "epoch": 0.09467455621301775, + "grad_norm": 0.4981457565578957, + "kl": 0.017822265625, + "learning_rate": 9.053254437869821e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 160 + }, + { + "completion_length": 119.0234375, + "epoch": 0.09526627218934912, + "grad_norm": 0.5076217668293294, + "kl": 0.018310546875, + "learning_rate": 9.047337278106508e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 161 + }, + { + "completion_length": 119.3359375, + "epoch": 0.09585798816568047, + "grad_norm": 0.38426698827175265, + "kl": 0.0186767578125, + "learning_rate": 9.041420118343195e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 162 + }, + { + "completion_length": 121.15625, + "epoch": 0.09644970414201183, + "grad_norm": 0.3354492174893896, + "kl": 0.0218505859375, + "learning_rate": 9.035502958579882e-07, + "loss": 0.0009, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 163 + }, + { + "completion_length": 137.3203125, + "epoch": 0.0970414201183432, + "grad_norm": 0.6003666177453194, + "kl": 0.0159912109375, + "learning_rate": 9.029585798816568e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 164 + }, + { + "completion_length": 116.6484375, + "epoch": 0.09763313609467456, + "grad_norm": 0.5844912791413156, + "kl": 0.01513671875, + "learning_rate": 9.023668639053253e-07, + "loss": 0.0006, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 165 + }, + { + "completion_length": 122.3984375, + "epoch": 0.09822485207100591, + "grad_norm": 0.64791767255888, + "kl": 0.021728515625, + "learning_rate": 9.01775147928994e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 166 + }, + { + "completion_length": 121.078125, + "epoch": 0.09881656804733728, + "grad_norm": 0.06717958698768606, + "kl": 0.0128173828125, + "learning_rate": 9.011834319526627e-07, + "loss": 0.0005, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 167 + }, + { + "completion_length": 111.78125, + "epoch": 0.09940828402366864, + "grad_norm": 0.6514138216888914, + "kl": 0.0159912109375, + "learning_rate": 9.005917159763313e-07, + "loss": 0.0006, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 168 + }, + { + "completion_length": 128.65625, + "epoch": 0.1, + "grad_norm": 0.5385241117358942, + "kl": 0.01446533203125, + "learning_rate": 9e-07, + "loss": 0.0006, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 169 + }, + { + "completion_length": 117.3125, + "epoch": 0.10059171597633136, + "grad_norm": 0.3363421071826952, + "kl": 0.018798828125, + "learning_rate": 8.994082840236686e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 170 + }, + { + "completion_length": 115.8671875, + "epoch": 0.10118343195266272, + "grad_norm": 0.7184816753818426, + "kl": 0.019775390625, + "learning_rate": 8.988165680473372e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 171 + }, + { + "completion_length": 121.3203125, + "epoch": 0.10177514792899409, + "grad_norm": 0.4130354226648162, + "kl": 0.01483154296875, + "learning_rate": 8.982248520710058e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 172 + }, + { + "completion_length": 109.2109375, + "epoch": 0.10236686390532544, + "grad_norm": 0.47089523938529726, + "kl": 0.0155029296875, + "learning_rate": 8.976331360946745e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 173 + }, + { + "completion_length": 115.03125, + "epoch": 0.10295857988165681, + "grad_norm": 0.8691184779288598, + "kl": 0.0184326171875, + "learning_rate": 8.970414201183432e-07, + "loss": 0.0007, + "reward": 0.0625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 174 + }, + { + "completion_length": 128.8671875, + "epoch": 0.10355029585798817, + "grad_norm": 0.40884113096481695, + "kl": 0.01470947265625, + "learning_rate": 8.964497041420118e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 175 + }, + { + "completion_length": 111.0703125, + "epoch": 0.10414201183431952, + "grad_norm": 1.8470551312333907, + "kl": 0.02392578125, + "learning_rate": 8.958579881656805e-07, + "loss": 0.001, + "reward": 0.171875, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 176 + }, + { + "completion_length": 105.1953125, + "epoch": 0.10473372781065089, + "grad_norm": 0.4936224946491018, + "kl": 0.015625, + "learning_rate": 8.95266272189349e-07, + "loss": 0.0006, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 177 + }, + { + "completion_length": 102.03125, + "epoch": 0.10532544378698225, + "grad_norm": 0.4775181425304535, + "kl": 0.01611328125, + "learning_rate": 8.946745562130177e-07, + "loss": 0.0006, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 178 + }, + { + "completion_length": 117.484375, + "epoch": 0.1059171597633136, + "grad_norm": 0.42237905517315566, + "kl": 0.012939453125, + "learning_rate": 8.940828402366864e-07, + "loss": 0.0005, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 179 + }, + { + "completion_length": 114.3671875, + "epoch": 0.10650887573964497, + "grad_norm": 0.4416742247121629, + "kl": 0.0123291015625, + "learning_rate": 8.93491124260355e-07, + "loss": 0.0005, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 180 + }, + { + "completion_length": 117.5625, + "epoch": 0.10710059171597633, + "grad_norm": 0.554325833734933, + "kl": 0.0205078125, + "learning_rate": 8.928994082840236e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 181 + }, + { + "completion_length": 122.4453125, + "epoch": 0.1076923076923077, + "grad_norm": 0.6257912718350611, + "kl": 0.01422119140625, + "learning_rate": 8.923076923076923e-07, + "loss": 0.0006, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 182 + }, + { + "completion_length": 111.1484375, + "epoch": 0.10828402366863905, + "grad_norm": 0.8550996505839848, + "kl": 0.0174560546875, + "learning_rate": 8.917159763313609e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 183 + }, + { + "completion_length": 114.828125, + "epoch": 0.10887573964497041, + "grad_norm": 0.4867095156638906, + "kl": 0.017822265625, + "learning_rate": 8.911242603550296e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 184 + }, + { + "completion_length": 114.015625, + "epoch": 0.10946745562130178, + "grad_norm": 0.5509968683809361, + "kl": 0.0208740234375, + "learning_rate": 8.905325443786981e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 185 + }, + { + "completion_length": 115.6640625, + "epoch": 0.11005917159763313, + "grad_norm": 0.6331723279504041, + "kl": 0.01483154296875, + "learning_rate": 8.899408284023668e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 186 + }, + { + "completion_length": 121.2265625, + "epoch": 0.11065088757396449, + "grad_norm": 0.43306572954147854, + "kl": 0.017578125, + "learning_rate": 8.893491124260355e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 187 + }, + { + "completion_length": 112.828125, + "epoch": 0.11124260355029586, + "grad_norm": 0.6084501672689493, + "kl": 0.026611328125, + "learning_rate": 8.887573964497042e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 188 + }, + { + "completion_length": 113.03125, + "epoch": 0.11183431952662722, + "grad_norm": 0.6654973684116855, + "kl": 0.020751953125, + "learning_rate": 8.881656804733727e-07, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 189 + }, + { + "completion_length": 125.1640625, + "epoch": 0.11242603550295859, + "grad_norm": 1.6862855249472077, + "kl": 0.019775390625, + "learning_rate": 8.875739644970413e-07, + "loss": 0.0008, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 190 + }, + { + "completion_length": 127.8515625, + "epoch": 0.11301775147928994, + "grad_norm": 0.10678934396143193, + "kl": 0.0166015625, + "learning_rate": 8.8698224852071e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 191 + }, + { + "completion_length": 110.4296875, + "epoch": 0.1136094674556213, + "grad_norm": 0.6732502769623764, + "kl": 0.0186767578125, + "learning_rate": 8.863905325443787e-07, + "loss": 0.0007, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 192 + }, + { + "completion_length": 128.953125, + "epoch": 0.11420118343195267, + "grad_norm": 0.10094269800951164, + "kl": 0.016357421875, + "learning_rate": 8.857988165680473e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 193 + }, + { + "completion_length": 111.1484375, + "epoch": 0.11479289940828402, + "grad_norm": 0.3400269323362785, + "kl": 0.0159912109375, + "learning_rate": 8.852071005917159e-07, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 194 + }, + { + "completion_length": 127.71875, + "epoch": 0.11538461538461539, + "grad_norm": 0.09907886578099627, + "kl": 0.0224609375, + "learning_rate": 8.846153846153846e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 195 + }, + { + "completion_length": 113.7421875, + "epoch": 0.11597633136094675, + "grad_norm": 0.7113567832329853, + "kl": 0.018310546875, + "learning_rate": 8.840236686390532e-07, + "loss": 0.0007, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 196 + }, + { + "completion_length": 118.1328125, + "epoch": 0.1165680473372781, + "grad_norm": 1.1633915254586764, + "kl": 0.0184326171875, + "learning_rate": 8.834319526627218e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 197 + }, + { + "completion_length": 118.375, + "epoch": 0.11715976331360947, + "grad_norm": 0.5293590347916067, + "kl": 0.01806640625, + "learning_rate": 8.828402366863905e-07, + "loss": 0.0007, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 198 + }, + { + "completion_length": 111.3203125, + "epoch": 0.11775147928994083, + "grad_norm": 0.652575880945242, + "kl": 0.019775390625, + "learning_rate": 8.822485207100592e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 199 + }, + { + "completion_length": 114.171875, + "epoch": 0.11834319526627218, + "grad_norm": 0.6257840929197565, + "kl": 0.01904296875, + "learning_rate": 8.816568047337278e-07, + "loss": 0.0008, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 200 + }, + { + "completion_length": 117.3828125, + "epoch": 0.11893491124260355, + "grad_norm": 0.5164526726281726, + "kl": 0.0189208984375, + "learning_rate": 8.810650887573963e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 201 + }, + { + "completion_length": 114.265625, + "epoch": 0.11952662721893491, + "grad_norm": 0.08133912113879489, + "kl": 0.01611328125, + "learning_rate": 8.80473372781065e-07, + "loss": 0.0006, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 202 + }, + { + "completion_length": 112.5859375, + "epoch": 0.12011834319526628, + "grad_norm": 0.3824909522840431, + "kl": 0.016845703125, + "learning_rate": 8.798816568047337e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 203 + }, + { + "completion_length": 128.4453125, + "epoch": 0.12071005917159763, + "grad_norm": 0.42006877855067803, + "kl": 0.0191650390625, + "learning_rate": 8.792899408284024e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 204 + }, + { + "completion_length": 103.0234375, + "epoch": 0.12130177514792899, + "grad_norm": 1.0066614426886888, + "kl": 0.0262451171875, + "learning_rate": 8.786982248520711e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 205 + }, + { + "completion_length": 116.5, + "epoch": 0.12189349112426036, + "grad_norm": 0.5108395005529607, + "kl": 0.021728515625, + "learning_rate": 8.781065088757395e-07, + "loss": 0.0009, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 206 + }, + { + "completion_length": 108.390625, + "epoch": 0.12248520710059171, + "grad_norm": 0.4173057636049301, + "kl": 0.01611328125, + "learning_rate": 8.775147928994082e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 207 + }, + { + "completion_length": 115.625, + "epoch": 0.12307692307692308, + "grad_norm": 0.39394930251141164, + "kl": 0.0213623046875, + "learning_rate": 8.769230769230769e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 208 + }, + { + "completion_length": 115.4453125, + "epoch": 0.12366863905325444, + "grad_norm": 0.6825603715626403, + "kl": 0.0166015625, + "learning_rate": 8.763313609467456e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 209 + }, + { + "completion_length": 108.0, + "epoch": 0.1242603550295858, + "grad_norm": 0.35890186032617116, + "kl": 0.0194091796875, + "learning_rate": 8.757396449704142e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 210 + }, + { + "completion_length": 117.2421875, + "epoch": 0.12485207100591716, + "grad_norm": 0.5908060472791105, + "kl": 0.0186767578125, + "learning_rate": 8.751479289940828e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 211 + }, + { + "completion_length": 122.6171875, + "epoch": 0.12544378698224853, + "grad_norm": 0.35679057359284433, + "kl": 0.01531982421875, + "learning_rate": 8.745562130177514e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 212 + }, + { + "completion_length": 110.0703125, + "epoch": 0.12603550295857988, + "grad_norm": 0.4652841654032183, + "kl": 0.017822265625, + "learning_rate": 8.739644970414201e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 213 + }, + { + "completion_length": 114.4453125, + "epoch": 0.12662721893491125, + "grad_norm": 0.48730119858702065, + "kl": 0.017333984375, + "learning_rate": 8.733727810650887e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 214 + }, + { + "completion_length": 124.4375, + "epoch": 0.12721893491124261, + "grad_norm": 0.27883165562392304, + "kl": 0.018310546875, + "learning_rate": 8.727810650887574e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 215 + }, + { + "completion_length": 113.40625, + "epoch": 0.12781065088757396, + "grad_norm": 1.5817571199427627, + "kl": 0.02197265625, + "learning_rate": 8.72189349112426e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 216 + }, + { + "completion_length": 115.5859375, + "epoch": 0.12840236686390533, + "grad_norm": 0.692005100384517, + "kl": 0.019287109375, + "learning_rate": 8.715976331360947e-07, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 217 + }, + { + "completion_length": 112.359375, + "epoch": 0.1289940828402367, + "grad_norm": 0.32781326629749014, + "kl": 0.0184326171875, + "learning_rate": 8.710059171597632e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 218 + }, + { + "completion_length": 127.25, + "epoch": 0.12958579881656804, + "grad_norm": 0.9686914629577781, + "kl": 0.024169921875, + "learning_rate": 8.704142011834319e-07, + "loss": 0.001, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 219 + }, + { + "completion_length": 118.0546875, + "epoch": 0.1301775147928994, + "grad_norm": 0.5470630680496033, + "kl": 0.023681640625, + "learning_rate": 8.698224852071006e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 220 + }, + { + "completion_length": 117.984375, + "epoch": 0.13076923076923078, + "grad_norm": 0.08112172732096905, + "kl": 0.018310546875, + "learning_rate": 8.692307692307692e-07, + "loss": 0.0007, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 221 + }, + { + "completion_length": 123.6484375, + "epoch": 0.13136094674556212, + "grad_norm": 0.6396401932902503, + "kl": 0.01904296875, + "learning_rate": 8.686390532544378e-07, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 222 + }, + { + "completion_length": 122.265625, + "epoch": 0.1319526627218935, + "grad_norm": 0.48984158684428375, + "kl": 0.01513671875, + "learning_rate": 8.680473372781064e-07, + "loss": 0.0006, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 223 + }, + { + "completion_length": 123.9140625, + "epoch": 0.13254437869822486, + "grad_norm": 0.3078656337907411, + "kl": 0.0157470703125, + "learning_rate": 8.674556213017751e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 224 + }, + { + "completion_length": 125.96875, + "epoch": 0.13313609467455623, + "grad_norm": 0.5756856163824874, + "kl": 0.01287841796875, + "learning_rate": 8.668639053254438e-07, + "loss": 0.0005, + "reward": 0.0234375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 225 + }, + { + "completion_length": 119.2890625, + "epoch": 0.13372781065088757, + "grad_norm": 0.7394001418666998, + "kl": 0.0213623046875, + "learning_rate": 8.662721893491123e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 226 + }, + { + "completion_length": 112.8828125, + "epoch": 0.13431952662721894, + "grad_norm": 0.08044314985022648, + "kl": 0.0186767578125, + "learning_rate": 8.65680473372781e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 227 + }, + { + "completion_length": 127.4765625, + "epoch": 0.1349112426035503, + "grad_norm": 0.5854798343114483, + "kl": 0.01409912109375, + "learning_rate": 8.650887573964497e-07, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 228 + }, + { + "completion_length": 122.2890625, + "epoch": 0.13550295857988165, + "grad_norm": 0.06604030594094341, + "kl": 0.0115966796875, + "learning_rate": 8.644970414201183e-07, + "loss": 0.0005, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 229 + }, + { + "completion_length": 121.90625, + "epoch": 0.13609467455621302, + "grad_norm": 0.4182810946058394, + "kl": 0.0198974609375, + "learning_rate": 8.639053254437869e-07, + "loss": 0.0008, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 230 + }, + { + "completion_length": 120.8046875, + "epoch": 0.1366863905325444, + "grad_norm": 1.0533541381731897, + "kl": 0.02294921875, + "learning_rate": 8.633136094674555e-07, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 231 + }, + { + "completion_length": 124.625, + "epoch": 0.13727810650887573, + "grad_norm": 0.06557675305705564, + "kl": 0.01434326171875, + "learning_rate": 8.627218934911242e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 232 + }, + { + "completion_length": 124.5625, + "epoch": 0.1378698224852071, + "grad_norm": 0.3277015586085132, + "kl": 0.0150146484375, + "learning_rate": 8.621301775147929e-07, + "loss": 0.0006, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 233 + }, + { + "completion_length": 126.203125, + "epoch": 0.13846153846153847, + "grad_norm": 0.39656780420845944, + "kl": 0.0162353515625, + "learning_rate": 8.615384615384616e-07, + "loss": 0.0007, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 234 + }, + { + "completion_length": 128.96875, + "epoch": 0.1390532544378698, + "grad_norm": 0.49224868501631275, + "kl": 0.0211181640625, + "learning_rate": 8.609467455621301e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 235 + }, + { + "completion_length": 131.90625, + "epoch": 0.13964497041420118, + "grad_norm": 0.3217203486532237, + "kl": 0.02099609375, + "learning_rate": 8.603550295857988e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 236 + }, + { + "completion_length": 121.3828125, + "epoch": 0.14023668639053255, + "grad_norm": 0.31334422266546513, + "kl": 0.0220947265625, + "learning_rate": 8.597633136094674e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 237 + }, + { + "completion_length": 131.8046875, + "epoch": 0.1408284023668639, + "grad_norm": 0.5936632637119341, + "kl": 0.0166015625, + "learning_rate": 8.591715976331361e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 238 + }, + { + "completion_length": 122.4296875, + "epoch": 0.14142011834319526, + "grad_norm": 0.6286727105887114, + "kl": 0.02001953125, + "learning_rate": 8.585798816568047e-07, + "loss": 0.0008, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 239 + }, + { + "completion_length": 125.6640625, + "epoch": 0.14201183431952663, + "grad_norm": 0.32587401155718193, + "kl": 0.0159912109375, + "learning_rate": 8.579881656804734e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 240 + }, + { + "completion_length": 120.0078125, + "epoch": 0.142603550295858, + "grad_norm": 0.802923133082496, + "kl": 0.0174560546875, + "learning_rate": 8.57396449704142e-07, + "loss": 0.0007, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 241 + }, + { + "completion_length": 122.9375, + "epoch": 0.14319526627218934, + "grad_norm": 0.3406872229323078, + "kl": 0.01513671875, + "learning_rate": 8.568047337278106e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 242 + }, + { + "completion_length": 126.0703125, + "epoch": 0.1437869822485207, + "grad_norm": 0.5497996110916782, + "kl": 0.01904296875, + "learning_rate": 8.562130177514792e-07, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 243 + }, + { + "completion_length": 118.2734375, + "epoch": 0.14437869822485208, + "grad_norm": 0.5681769068910806, + "kl": 0.01611328125, + "learning_rate": 8.556213017751479e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 244 + }, + { + "completion_length": 122.03125, + "epoch": 0.14497041420118342, + "grad_norm": 0.13359058101234492, + "kl": 0.01348876953125, + "learning_rate": 8.550295857988166e-07, + "loss": 0.0005, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 245 + }, + { + "completion_length": 113.5390625, + "epoch": 0.1455621301775148, + "grad_norm": 0.6191866572469173, + "kl": 0.0157470703125, + "learning_rate": 8.544378698224852e-07, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 246 + }, + { + "completion_length": 123.34375, + "epoch": 0.14615384615384616, + "grad_norm": 0.7820173295440371, + "kl": 0.0218505859375, + "learning_rate": 8.538461538461537e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 247 + }, + { + "completion_length": 126.5546875, + "epoch": 0.1467455621301775, + "grad_norm": 0.8199960657752614, + "kl": 0.02197265625, + "learning_rate": 8.532544378698224e-07, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 248 + }, + { + "completion_length": 120.421875, + "epoch": 0.14733727810650887, + "grad_norm": 0.8609608523523595, + "kl": 0.01953125, + "learning_rate": 8.526627218934911e-07, + "loss": 0.0008, + "reward": 0.09375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 249 + }, + { + "completion_length": 121.6015625, + "epoch": 0.14792899408284024, + "grad_norm": 0.4873240829738657, + "kl": 0.02197265625, + "learning_rate": 8.520710059171598e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 250 + }, + { + "completion_length": 127.40625, + "epoch": 0.14852071005917158, + "grad_norm": 0.31471946228349534, + "kl": 0.01434326171875, + "learning_rate": 8.514792899408284e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 251 + }, + { + "completion_length": 126.4453125, + "epoch": 0.14911242603550295, + "grad_norm": 0.398606932697859, + "kl": 0.015380859375, + "learning_rate": 8.508875739644969e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 252 + }, + { + "completion_length": 116.328125, + "epoch": 0.14970414201183432, + "grad_norm": 3.30352861169374, + "kl": 0.019287109375, + "learning_rate": 8.502958579881656e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 253 + }, + { + "completion_length": 119.7109375, + "epoch": 0.1502958579881657, + "grad_norm": 0.6583908226632988, + "kl": 0.0174560546875, + "learning_rate": 8.497041420118343e-07, + "loss": 0.0007, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 254 + }, + { + "completion_length": 125.8671875, + "epoch": 0.15088757396449703, + "grad_norm": 0.24184787085931278, + "kl": 0.01251220703125, + "learning_rate": 8.491124260355029e-07, + "loss": 0.0005, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 255 + }, + { + "completion_length": 115.96875, + "epoch": 0.1514792899408284, + "grad_norm": 0.4976592619887303, + "kl": 0.01611328125, + "learning_rate": 8.485207100591716e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 256 + }, + { + "completion_length": 121.796875, + "epoch": 0.15207100591715977, + "grad_norm": 0.43217319897851303, + "kl": 0.015869140625, + "learning_rate": 8.479289940828402e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 257 + }, + { + "completion_length": 121.796875, + "epoch": 0.15266272189349112, + "grad_norm": 0.4300499737877329, + "kl": 0.014892578125, + "learning_rate": 8.473372781065088e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 258 + }, + { + "completion_length": 119.8203125, + "epoch": 0.15325443786982249, + "grad_norm": 0.33435860397284156, + "kl": 0.01507568359375, + "learning_rate": 8.467455621301774e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 259 + }, + { + "completion_length": 124.2890625, + "epoch": 0.15384615384615385, + "grad_norm": 0.5510280207741802, + "kl": 0.016845703125, + "learning_rate": 8.461538461538461e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 260 + }, + { + "completion_length": 124.125, + "epoch": 0.1544378698224852, + "grad_norm": 0.40141325978365006, + "kl": 0.018310546875, + "learning_rate": 8.455621301775148e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 261 + }, + { + "completion_length": 110.21875, + "epoch": 0.15502958579881657, + "grad_norm": 0.9265274138807579, + "kl": 0.0264892578125, + "learning_rate": 8.449704142011834e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 262 + }, + { + "completion_length": 112.1875, + "epoch": 0.15562130177514794, + "grad_norm": 0.06595940288131931, + "kl": 0.0162353515625, + "learning_rate": 8.443786982248521e-07, + "loss": 0.0007, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 263 + }, + { + "completion_length": 129.7109375, + "epoch": 0.15621301775147928, + "grad_norm": 0.08783846771106694, + "kl": 0.0196533203125, + "learning_rate": 8.437869822485206e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 264 + }, + { + "completion_length": 123.46875, + "epoch": 0.15680473372781065, + "grad_norm": 0.23717419491581296, + "kl": 0.01904296875, + "learning_rate": 8.431952662721893e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 265 + }, + { + "completion_length": 113.4765625, + "epoch": 0.15739644970414202, + "grad_norm": 0.30755577748949464, + "kl": 0.02392578125, + "learning_rate": 8.42603550295858e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 266 + }, + { + "completion_length": 116.25, + "epoch": 0.15798816568047339, + "grad_norm": 0.6559627193560177, + "kl": 0.018310546875, + "learning_rate": 8.420118343195267e-07, + "loss": 0.0007, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 267 + }, + { + "completion_length": 115.2578125, + "epoch": 0.15857988165680473, + "grad_norm": 0.8751023061806841, + "kl": 0.018798828125, + "learning_rate": 8.414201183431952e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 268 + }, + { + "completion_length": 120.1171875, + "epoch": 0.1591715976331361, + "grad_norm": 0.5951457740139074, + "kl": 0.0238037109375, + "learning_rate": 8.408284023668639e-07, + "loss": 0.001, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 269 + }, + { + "completion_length": 114.8515625, + "epoch": 0.15976331360946747, + "grad_norm": 1.4766099792317464, + "kl": 0.02099609375, + "learning_rate": 8.402366863905325e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 270 + }, + { + "completion_length": 125.765625, + "epoch": 0.1603550295857988, + "grad_norm": 1.0476419835888864, + "kl": 0.02587890625, + "learning_rate": 8.396449704142012e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 271 + }, + { + "completion_length": 115.578125, + "epoch": 0.16094674556213018, + "grad_norm": 0.3832899413612094, + "kl": 0.0203857421875, + "learning_rate": 8.390532544378697e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 272 + }, + { + "completion_length": 106.1171875, + "epoch": 0.16153846153846155, + "grad_norm": 1.0372057360684541, + "kl": 0.0291748046875, + "learning_rate": 8.384615384615384e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 273 + }, + { + "completion_length": 116.2890625, + "epoch": 0.1621301775147929, + "grad_norm": 0.7916508577416639, + "kl": 0.020263671875, + "learning_rate": 8.378698224852071e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 274 + }, + { + "completion_length": 113.546875, + "epoch": 0.16272189349112426, + "grad_norm": 0.4081254923691964, + "kl": 0.0159912109375, + "learning_rate": 8.372781065088757e-07, + "loss": 0.0006, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 275 + }, + { + "completion_length": 113.9453125, + "epoch": 0.16331360946745563, + "grad_norm": 0.3973003495347372, + "kl": 0.01513671875, + "learning_rate": 8.366863905325443e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 276 + }, + { + "completion_length": 113.4921875, + "epoch": 0.16390532544378697, + "grad_norm": 0.39961454747005926, + "kl": 0.021484375, + "learning_rate": 8.36094674556213e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 277 + }, + { + "completion_length": 114.34375, + "epoch": 0.16449704142011834, + "grad_norm": 0.889849972735649, + "kl": 0.025146484375, + "learning_rate": 8.355029585798816e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 278 + }, + { + "completion_length": 106.6640625, + "epoch": 0.1650887573964497, + "grad_norm": 0.7551292612190929, + "kl": 0.01904296875, + "learning_rate": 8.349112426035503e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 279 + }, + { + "completion_length": 122.203125, + "epoch": 0.16568047337278108, + "grad_norm": 0.5938342895992142, + "kl": 0.0218505859375, + "learning_rate": 8.343195266272189e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 280 + }, + { + "completion_length": 111.0859375, + "epoch": 0.16627218934911242, + "grad_norm": 0.5450448672366255, + "kl": 0.0257568359375, + "learning_rate": 8.337278106508875e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 281 + }, + { + "completion_length": 115.25, + "epoch": 0.1668639053254438, + "grad_norm": 0.6369356100951221, + "kl": 0.021484375, + "learning_rate": 8.331360946745562e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 282 + }, + { + "completion_length": 122.0, + "epoch": 0.16745562130177516, + "grad_norm": 0.575368571185395, + "kl": 0.0216064453125, + "learning_rate": 8.325443786982248e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 283 + }, + { + "completion_length": 118.8515625, + "epoch": 0.1680473372781065, + "grad_norm": 0.942819826911523, + "kl": 0.023193359375, + "learning_rate": 8.319526627218934e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 284 + }, + { + "completion_length": 108.7734375, + "epoch": 0.16863905325443787, + "grad_norm": 0.2942616169495444, + "kl": 0.018798828125, + "learning_rate": 8.313609467455621e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 285 + }, + { + "completion_length": 112.28125, + "epoch": 0.16923076923076924, + "grad_norm": 0.2872078782112408, + "kl": 0.0205078125, + "learning_rate": 8.307692307692308e-07, + "loss": 0.0008, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 286 + }, + { + "completion_length": 119.890625, + "epoch": 0.16982248520710058, + "grad_norm": 0.4766843567617003, + "kl": 0.018798828125, + "learning_rate": 8.301775147928994e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 287 + }, + { + "completion_length": 117.2421875, + "epoch": 0.17041420118343195, + "grad_norm": 0.5713400310515454, + "kl": 0.0194091796875, + "learning_rate": 8.295857988165679e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 288 + }, + { + "completion_length": 115.21875, + "epoch": 0.17100591715976332, + "grad_norm": 0.746931685681302, + "kl": 0.027099609375, + "learning_rate": 8.289940828402366e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.109375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 289 + }, + { + "completion_length": 124.1953125, + "epoch": 0.17159763313609466, + "grad_norm": 1.2866378152091364, + "kl": 0.0272216796875, + "learning_rate": 8.284023668639053e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 290 + }, + { + "completion_length": 106.421875, + "epoch": 0.17218934911242603, + "grad_norm": 0.7970594025474053, + "kl": 0.021728515625, + "learning_rate": 8.27810650887574e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 291 + }, + { + "completion_length": 117.9140625, + "epoch": 0.1727810650887574, + "grad_norm": 0.3148061699379425, + "kl": 0.0205078125, + "learning_rate": 8.272189349112427e-07, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 292 + }, + { + "completion_length": 115.1328125, + "epoch": 0.17337278106508874, + "grad_norm": 0.4740182243842568, + "kl": 0.021484375, + "learning_rate": 8.266272189349111e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 293 + }, + { + "completion_length": 119.9375, + "epoch": 0.1739644970414201, + "grad_norm": 0.5647176312603209, + "kl": 0.02880859375, + "learning_rate": 8.260355029585798e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 294 + }, + { + "completion_length": 116.9921875, + "epoch": 0.17455621301775148, + "grad_norm": 1.5198009546445463, + "kl": 0.0223388671875, + "learning_rate": 8.254437869822485e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 295 + }, + { + "completion_length": 113.234375, + "epoch": 0.17514792899408285, + "grad_norm": 1.050637073499939, + "kl": 0.0177001953125, + "learning_rate": 8.248520710059172e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 296 + }, + { + "completion_length": 114.625, + "epoch": 0.1757396449704142, + "grad_norm": 0.7115465254070188, + "kl": 0.0255126953125, + "learning_rate": 8.242603550295858e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 297 + }, + { + "completion_length": 114.9921875, + "epoch": 0.17633136094674556, + "grad_norm": 0.46554467972603547, + "kl": 0.023193359375, + "learning_rate": 8.236686390532544e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 298 + }, + { + "completion_length": 111.890625, + "epoch": 0.17692307692307693, + "grad_norm": 0.43846497007578344, + "kl": 0.0213623046875, + "learning_rate": 8.23076923076923e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 299 + }, + { + "completion_length": 116.46875, + "epoch": 0.17751479289940827, + "grad_norm": 3.009978073294122, + "kl": 0.03125, + "learning_rate": 8.224852071005917e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 300 + }, + { + "completion_length": 108.125, + "epoch": 0.17810650887573964, + "grad_norm": 0.3870144601432239, + "kl": 0.024169921875, + "learning_rate": 8.218934911242603e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 301 + }, + { + "completion_length": 122.0859375, + "epoch": 0.178698224852071, + "grad_norm": 0.7117722776557506, + "kl": 0.02490234375, + "learning_rate": 8.21301775147929e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 302 + }, + { + "completion_length": 115.9609375, + "epoch": 0.17928994082840236, + "grad_norm": 0.3620320426822417, + "kl": 0.0166015625, + "learning_rate": 8.207100591715976e-07, + "loss": 0.0007, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 303 + }, + { + "completion_length": 113.6953125, + "epoch": 0.17988165680473372, + "grad_norm": 0.9298277910536983, + "kl": 0.032470703125, + "learning_rate": 8.201183431952662e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 304 + }, + { + "completion_length": 116.34375, + "epoch": 0.1804733727810651, + "grad_norm": 0.5897774438548435, + "kl": 0.021240234375, + "learning_rate": 8.195266272189348e-07, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 305 + }, + { + "completion_length": 114.6875, + "epoch": 0.18106508875739644, + "grad_norm": 0.7776681851606145, + "kl": 0.026123046875, + "learning_rate": 8.189349112426035e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 306 + }, + { + "completion_length": 121.8671875, + "epoch": 0.1816568047337278, + "grad_norm": 0.4577361701856228, + "kl": 0.02392578125, + "learning_rate": 8.183431952662722e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 307 + }, + { + "completion_length": 115.671875, + "epoch": 0.18224852071005918, + "grad_norm": 0.0846411613238198, + "kl": 0.0234375, + "learning_rate": 8.177514792899409e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 308 + }, + { + "completion_length": 126.71875, + "epoch": 0.18284023668639054, + "grad_norm": 5.338386360100844, + "kl": 0.032470703125, + "learning_rate": 8.171597633136094e-07, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 309 + }, + { + "completion_length": 115.9765625, + "epoch": 0.1834319526627219, + "grad_norm": 0.6158112458534385, + "kl": 0.02734375, + "learning_rate": 8.16568047337278e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 310 + }, + { + "completion_length": 114.953125, + "epoch": 0.18402366863905326, + "grad_norm": 0.6440002720772642, + "kl": 0.03173828125, + "learning_rate": 8.159763313609467e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 311 + }, + { + "completion_length": 111.109375, + "epoch": 0.18461538461538463, + "grad_norm": 1.0269169408082788, + "kl": 0.0322265625, + "learning_rate": 8.153846153846154e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 312 + }, + { + "completion_length": 119.34375, + "epoch": 0.18520710059171597, + "grad_norm": 0.3624425858160388, + "kl": 0.0191650390625, + "learning_rate": 8.14792899408284e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 313 + }, + { + "completion_length": 123.8984375, + "epoch": 0.18579881656804734, + "grad_norm": 1.4038659675583196, + "kl": 0.02880859375, + "learning_rate": 8.142011834319526e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 314 + }, + { + "completion_length": 116.5546875, + "epoch": 0.1863905325443787, + "grad_norm": 0.6727516785944477, + "kl": 0.036376953125, + "learning_rate": 8.136094674556213e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 315 + }, + { + "completion_length": 122.9921875, + "epoch": 0.18698224852071005, + "grad_norm": 0.25755647476243093, + "kl": 0.019775390625, + "learning_rate": 8.130177514792899e-07, + "loss": 0.0008, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 316 + }, + { + "completion_length": 117.6328125, + "epoch": 0.18757396449704142, + "grad_norm": 0.47333826024803555, + "kl": 0.025390625, + "learning_rate": 8.124260355029585e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 317 + }, + { + "completion_length": 115.859375, + "epoch": 0.1881656804733728, + "grad_norm": 0.5502578272689622, + "kl": 0.02685546875, + "learning_rate": 8.118343195266272e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 318 + }, + { + "completion_length": 119.34375, + "epoch": 0.18875739644970413, + "grad_norm": 0.6632456478885603, + "kl": 0.0284423828125, + "learning_rate": 8.112426035502958e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 319 + }, + { + "completion_length": 118.0078125, + "epoch": 0.1893491124260355, + "grad_norm": 1.3250433273198754, + "kl": 0.04296875, + "learning_rate": 8.106508875739645e-07, + "loss": 0.0017, + "reward": 0.2109375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2109375, + "step": 320 + }, + { + "completion_length": 115.109375, + "epoch": 0.18994082840236687, + "grad_norm": 0.8221564383051803, + "kl": 0.031005859375, + "learning_rate": 8.100591715976332e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 321 + }, + { + "completion_length": 123.4921875, + "epoch": 0.19053254437869824, + "grad_norm": 0.07650554311039742, + "kl": 0.0260009765625, + "learning_rate": 8.094674556213017e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 322 + }, + { + "completion_length": 122.40625, + "epoch": 0.19112426035502958, + "grad_norm": 0.7761766067685122, + "kl": 0.0299072265625, + "learning_rate": 8.088757396449704e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 323 + }, + { + "completion_length": 119.921875, + "epoch": 0.19171597633136095, + "grad_norm": 0.3777040835281499, + "kl": 0.01953125, + "learning_rate": 8.08284023668639e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 324 + }, + { + "completion_length": 123.7890625, + "epoch": 0.19230769230769232, + "grad_norm": 0.7020149035742685, + "kl": 0.02734375, + "learning_rate": 8.076923076923077e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 325 + }, + { + "completion_length": 120.3984375, + "epoch": 0.19289940828402366, + "grad_norm": 4.741423014005859, + "kl": 0.052734375, + "learning_rate": 8.071005917159763e-07, + "loss": 0.0021, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 326 + }, + { + "completion_length": 117.484375, + "epoch": 0.19349112426035503, + "grad_norm": 0.45428752746835055, + "kl": 0.025390625, + "learning_rate": 8.06508875739645e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 327 + }, + { + "completion_length": 126.2265625, + "epoch": 0.1940828402366864, + "grad_norm": 0.8464696205160708, + "kl": 0.03271484375, + "learning_rate": 8.059171597633136e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 328 + }, + { + "completion_length": 114.0078125, + "epoch": 0.19467455621301774, + "grad_norm": 0.8727151528947891, + "kl": 0.03564453125, + "learning_rate": 8.053254437869822e-07, + "loss": 0.0014, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 329 + }, + { + "completion_length": 125.140625, + "epoch": 0.1952662721893491, + "grad_norm": 0.2809724927024228, + "kl": 0.0225830078125, + "learning_rate": 8.047337278106508e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 330 + }, + { + "completion_length": 116.3203125, + "epoch": 0.19585798816568048, + "grad_norm": 0.5794854274037563, + "kl": 0.02734375, + "learning_rate": 8.041420118343195e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 331 + }, + { + "completion_length": 124.1484375, + "epoch": 0.19644970414201182, + "grad_norm": 0.4484856815759359, + "kl": 0.02197265625, + "learning_rate": 8.035502958579882e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 332 + }, + { + "completion_length": 122.890625, + "epoch": 0.1970414201183432, + "grad_norm": 0.36964519971421256, + "kl": 0.025146484375, + "learning_rate": 8.029585798816568e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 333 + }, + { + "completion_length": 119.5703125, + "epoch": 0.19763313609467456, + "grad_norm": 0.4031681657360233, + "kl": 0.0250244140625, + "learning_rate": 8.023668639053253e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 334 + }, + { + "completion_length": 123.546875, + "epoch": 0.19822485207100593, + "grad_norm": 1.1002607094351988, + "kl": 0.0289306640625, + "learning_rate": 8.01775147928994e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 335 + }, + { + "completion_length": 132.40625, + "epoch": 0.19881656804733727, + "grad_norm": 0.3006899782300881, + "kl": 0.024658203125, + "learning_rate": 8.011834319526627e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 336 + }, + { + "completion_length": 125.8671875, + "epoch": 0.19940828402366864, + "grad_norm": 0.4220498960569091, + "kl": 0.02490234375, + "learning_rate": 8.005917159763314e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 337 + }, + { + "completion_length": 136.8359375, + "epoch": 0.2, + "grad_norm": 0.5082129796329056, + "kl": 0.0230712890625, + "learning_rate": 8e-07, + "loss": 0.0009, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 338 + }, + { + "completion_length": 136.1796875, + "epoch": 0.20059171597633135, + "grad_norm": 0.5936127420763803, + "kl": 0.018798828125, + "learning_rate": 7.994082840236685e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 339 + }, + { + "completion_length": 142.0546875, + "epoch": 0.20118343195266272, + "grad_norm": 0.4061150924734827, + "kl": 0.0177001953125, + "learning_rate": 7.988165680473372e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 340 + }, + { + "completion_length": 141.6875, + "epoch": 0.2017751479289941, + "grad_norm": 0.7826813932047468, + "kl": 0.0235595703125, + "learning_rate": 7.982248520710059e-07, + "loss": 0.0009, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 341 + }, + { + "completion_length": 120.9296875, + "epoch": 0.20236686390532543, + "grad_norm": 0.5418696788872945, + "kl": 0.02978515625, + "learning_rate": 7.976331360946745e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 342 + }, + { + "completion_length": 126.1171875, + "epoch": 0.2029585798816568, + "grad_norm": 0.5993040601306702, + "kl": 0.0213623046875, + "learning_rate": 7.970414201183432e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 343 + }, + { + "completion_length": 130.5859375, + "epoch": 0.20355029585798817, + "grad_norm": 0.6424684990332961, + "kl": 0.025634765625, + "learning_rate": 7.964497041420118e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 344 + }, + { + "completion_length": 137.0625, + "epoch": 0.20414201183431951, + "grad_norm": 0.5774064073267865, + "kl": 0.022216796875, + "learning_rate": 7.958579881656804e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 345 + }, + { + "completion_length": 134.8125, + "epoch": 0.20473372781065088, + "grad_norm": 0.3393828092039438, + "kl": 0.0172119140625, + "learning_rate": 7.95266272189349e-07, + "loss": 0.0007, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 346 + }, + { + "completion_length": 135.5234375, + "epoch": 0.20532544378698225, + "grad_norm": 0.5804650495923589, + "kl": 0.0245361328125, + "learning_rate": 7.946745562130177e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 347 + }, + { + "completion_length": 132.921875, + "epoch": 0.20591715976331362, + "grad_norm": 0.09544624676206989, + "kl": 0.0218505859375, + "learning_rate": 7.940828402366864e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 348 + }, + { + "completion_length": 145.8046875, + "epoch": 0.20650887573964496, + "grad_norm": 0.3692076761877462, + "kl": 0.0230712890625, + "learning_rate": 7.93491124260355e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 349 + }, + { + "completion_length": 134.4296875, + "epoch": 0.20710059171597633, + "grad_norm": 0.30824299294569424, + "kl": 0.027099609375, + "learning_rate": 7.928994082840237e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 350 + }, + { + "completion_length": 143.671875, + "epoch": 0.2076923076923077, + "grad_norm": 0.6354924464370045, + "kl": 0.0244140625, + "learning_rate": 7.923076923076922e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 351 + }, + { + "completion_length": 138.1484375, + "epoch": 0.20828402366863905, + "grad_norm": 0.364627175348422, + "kl": 0.0255126953125, + "learning_rate": 7.917159763313609e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 352 + }, + { + "completion_length": 141.3046875, + "epoch": 0.20887573964497042, + "grad_norm": 0.47597009066492796, + "kl": 0.025146484375, + "learning_rate": 7.911242603550296e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 353 + }, + { + "completion_length": 138.6953125, + "epoch": 0.20946745562130178, + "grad_norm": 0.5284613457410549, + "kl": 0.03759765625, + "learning_rate": 7.905325443786983e-07, + "loss": 0.0015, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 354 + }, + { + "completion_length": 141.5625, + "epoch": 0.21005917159763313, + "grad_norm": 0.2820100906675037, + "kl": 0.02099609375, + "learning_rate": 7.899408284023668e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 355 + }, + { + "completion_length": 152.1328125, + "epoch": 0.2106508875739645, + "grad_norm": 0.7633878911664203, + "kl": 0.02490234375, + "learning_rate": 7.893491124260354e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 356 + }, + { + "completion_length": 141.4609375, + "epoch": 0.21124260355029587, + "grad_norm": 0.35721311941119144, + "kl": 0.0213623046875, + "learning_rate": 7.887573964497041e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 357 + }, + { + "completion_length": 136.6953125, + "epoch": 0.2118343195266272, + "grad_norm": 0.5364543435861183, + "kl": 0.02392578125, + "learning_rate": 7.881656804733728e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 358 + }, + { + "completion_length": 140.7734375, + "epoch": 0.21242603550295858, + "grad_norm": 0.6986287105430898, + "kl": 0.0250244140625, + "learning_rate": 7.875739644970414e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 359 + }, + { + "completion_length": 138.765625, + "epoch": 0.21301775147928995, + "grad_norm": 0.4779020943144175, + "kl": 0.021240234375, + "learning_rate": 7.8698224852071e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 360 + }, + { + "completion_length": 132.953125, + "epoch": 0.2136094674556213, + "grad_norm": 0.588144917340528, + "kl": 0.0230712890625, + "learning_rate": 7.863905325443787e-07, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 361 + }, + { + "completion_length": 136.2890625, + "epoch": 0.21420118343195266, + "grad_norm": 0.5534854698524752, + "kl": 0.0211181640625, + "learning_rate": 7.857988165680473e-07, + "loss": 0.0008, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 362 + }, + { + "completion_length": 126.90625, + "epoch": 0.21479289940828403, + "grad_norm": 0.5817287036373449, + "kl": 0.0263671875, + "learning_rate": 7.852071005917159e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 363 + }, + { + "completion_length": 124.109375, + "epoch": 0.2153846153846154, + "grad_norm": 0.46379268816962277, + "kl": 0.02978515625, + "learning_rate": 7.846153846153846e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 364 + }, + { + "completion_length": 132.03125, + "epoch": 0.21597633136094674, + "grad_norm": 0.6118813025944961, + "kl": 0.018798828125, + "learning_rate": 7.840236686390532e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 365 + }, + { + "completion_length": 122.3984375, + "epoch": 0.2165680473372781, + "grad_norm": 0.1383206779315089, + "kl": 0.0224609375, + "learning_rate": 7.834319526627219e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 366 + }, + { + "completion_length": 128.578125, + "epoch": 0.21715976331360948, + "grad_norm": 0.5423446611942573, + "kl": 0.02587890625, + "learning_rate": 7.828402366863905e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 367 + }, + { + "completion_length": 127.9453125, + "epoch": 0.21775147928994082, + "grad_norm": 0.2791338939870674, + "kl": 0.015625, + "learning_rate": 7.822485207100591e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 368 + }, + { + "completion_length": 121.515625, + "epoch": 0.2183431952662722, + "grad_norm": 0.45810859714238156, + "kl": 0.0240478515625, + "learning_rate": 7.816568047337278e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 369 + }, + { + "completion_length": 123.46875, + "epoch": 0.21893491124260356, + "grad_norm": 0.5143020845821262, + "kl": 0.0211181640625, + "learning_rate": 7.810650887573964e-07, + "loss": 0.0008, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 370 + }, + { + "completion_length": 126.7265625, + "epoch": 0.2195266272189349, + "grad_norm": 0.6840200831951843, + "kl": 0.02294921875, + "learning_rate": 7.80473372781065e-07, + "loss": 0.0009, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 371 + }, + { + "completion_length": 127.484375, + "epoch": 0.22011834319526627, + "grad_norm": 0.39909324997528517, + "kl": 0.024658203125, + "learning_rate": 7.798816568047337e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 372 + }, + { + "completion_length": 122.609375, + "epoch": 0.22071005917159764, + "grad_norm": 0.9353800741661317, + "kl": 0.020751953125, + "learning_rate": 7.792899408284024e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 373 + }, + { + "completion_length": 119.75, + "epoch": 0.22130177514792898, + "grad_norm": 0.40171319965340596, + "kl": 0.02294921875, + "learning_rate": 7.78698224852071e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 374 + }, + { + "completion_length": 126.0234375, + "epoch": 0.22189349112426035, + "grad_norm": 0.42378454021223927, + "kl": 0.0216064453125, + "learning_rate": 7.781065088757395e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 375 + }, + { + "completion_length": 117.671875, + "epoch": 0.22248520710059172, + "grad_norm": 0.728284993745523, + "kl": 0.02685546875, + "learning_rate": 7.775147928994082e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 376 + }, + { + "completion_length": 112.640625, + "epoch": 0.2230769230769231, + "grad_norm": 1.0626713596807542, + "kl": 0.02392578125, + "learning_rate": 7.769230769230769e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.11904378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 377 + }, + { + "completion_length": 125.96875, + "epoch": 0.22366863905325443, + "grad_norm": 2.2243146711779973, + "kl": 0.025146484375, + "learning_rate": 7.763313609467456e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 378 + }, + { + "completion_length": 120.3515625, + "epoch": 0.2242603550295858, + "grad_norm": 0.3317948587559403, + "kl": 0.0220947265625, + "learning_rate": 7.757396449704143e-07, + "loss": 0.0009, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 379 + }, + { + "completion_length": 119.046875, + "epoch": 0.22485207100591717, + "grad_norm": 0.6300690906927161, + "kl": 0.021240234375, + "learning_rate": 7.751479289940827e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 380 + }, + { + "completion_length": 118.2265625, + "epoch": 0.2254437869822485, + "grad_norm": 0.6005778188328701, + "kl": 0.034423828125, + "learning_rate": 7.745562130177514e-07, + "loss": 0.0014, + "reward": 0.1484375, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 381 + }, + { + "completion_length": 117.5546875, + "epoch": 0.22603550295857988, + "grad_norm": 0.7472832150773161, + "kl": 0.028564453125, + "learning_rate": 7.739644970414201e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 382 + }, + { + "completion_length": 114.75, + "epoch": 0.22662721893491125, + "grad_norm": 2.309967237427885, + "kl": 0.029052734375, + "learning_rate": 7.733727810650888e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 383 + }, + { + "completion_length": 118.0859375, + "epoch": 0.2272189349112426, + "grad_norm": 0.889829352614745, + "kl": 0.0238037109375, + "learning_rate": 7.727810650887574e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.12983438372612, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.0859375, + "step": 384 + }, + { + "completion_length": 110.5, + "epoch": 0.22781065088757396, + "grad_norm": 0.11057691966786361, + "kl": 0.0283203125, + "learning_rate": 7.721893491124259e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 385 + }, + { + "completion_length": 118.3125, + "epoch": 0.22840236686390533, + "grad_norm": 0.9081092481429001, + "kl": 0.031005859375, + "learning_rate": 7.715976331360946e-07, + "loss": 0.0012, + "reward": 0.15625, + "reward_std": 0.11662659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 386 + }, + { + "completion_length": 124.046875, + "epoch": 0.22899408284023667, + "grad_norm": 0.6038999031657474, + "kl": 0.0322265625, + "learning_rate": 7.710059171597633e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 387 + }, + { + "completion_length": 122.6171875, + "epoch": 0.22958579881656804, + "grad_norm": 0.5488965358733676, + "kl": 0.0263671875, + "learning_rate": 7.704142011834319e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 388 + }, + { + "completion_length": 117.1640625, + "epoch": 0.2301775147928994, + "grad_norm": 0.679363765766454, + "kl": 0.0244140625, + "learning_rate": 7.698224852071006e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 389 + }, + { + "completion_length": 135.1875, + "epoch": 0.23076923076923078, + "grad_norm": 0.2882477563916687, + "kl": 0.028076171875, + "learning_rate": 7.692307692307693e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 390 + }, + { + "completion_length": 130.2578125, + "epoch": 0.23136094674556212, + "grad_norm": 0.6599763316734897, + "kl": 0.025634765625, + "learning_rate": 7.686390532544378e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 391 + }, + { + "completion_length": 131.6171875, + "epoch": 0.2319526627218935, + "grad_norm": 0.5124871282591077, + "kl": 0.0277099609375, + "learning_rate": 7.680473372781064e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 392 + }, + { + "completion_length": 125.734375, + "epoch": 0.23254437869822486, + "grad_norm": 0.9776970615448201, + "kl": 0.0264892578125, + "learning_rate": 7.674556213017751e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 393 + }, + { + "completion_length": 131.7890625, + "epoch": 0.2331360946745562, + "grad_norm": 0.590469429227314, + "kl": 0.02734375, + "learning_rate": 7.668639053254438e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 394 + }, + { + "completion_length": 131.203125, + "epoch": 0.23372781065088757, + "grad_norm": 0.38024518281248, + "kl": 0.0218505859375, + "learning_rate": 7.662721893491125e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 395 + }, + { + "completion_length": 129.3046875, + "epoch": 0.23431952662721894, + "grad_norm": 0.37935416807810435, + "kl": 0.033203125, + "learning_rate": 7.65680473372781e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 396 + }, + { + "completion_length": 125.9375, + "epoch": 0.23491124260355029, + "grad_norm": 0.8763377073515005, + "kl": 0.036865234375, + "learning_rate": 7.650887573964496e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 397 + }, + { + "completion_length": 125.609375, + "epoch": 0.23550295857988165, + "grad_norm": 0.2906467999806661, + "kl": 0.0303955078125, + "learning_rate": 7.644970414201183e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 398 + }, + { + "completion_length": 122.21875, + "epoch": 0.23609467455621302, + "grad_norm": 0.5661229720575791, + "kl": 0.034423828125, + "learning_rate": 7.63905325443787e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 399 + }, + { + "completion_length": 137.2109375, + "epoch": 0.23668639053254437, + "grad_norm": 0.28119402002878713, + "kl": 0.0262451171875, + "learning_rate": 7.633136094674556e-07, + "loss": 0.001, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 400 + }, + { + "completion_length": 131.7421875, + "epoch": 0.23727810650887574, + "grad_norm": 0.5796430742155207, + "kl": 0.035888671875, + "learning_rate": 7.627218934911242e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.07216878235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 401 + }, + { + "completion_length": 123.0, + "epoch": 0.2378698224852071, + "grad_norm": 0.36808438709706803, + "kl": 0.023193359375, + "learning_rate": 7.621301775147929e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 402 + }, + { + "completion_length": 126.4375, + "epoch": 0.23846153846153847, + "grad_norm": 0.6359918396519932, + "kl": 0.0289306640625, + "learning_rate": 7.615384615384615e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 403 + }, + { + "completion_length": 126.6171875, + "epoch": 0.23905325443786982, + "grad_norm": 0.39908845054940373, + "kl": 0.02490234375, + "learning_rate": 7.609467455621301e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 404 + }, + { + "completion_length": 131.7265625, + "epoch": 0.23964497041420119, + "grad_norm": 0.9642224957600413, + "kl": 0.02880859375, + "learning_rate": 7.603550295857988e-07, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.109375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 405 + }, + { + "completion_length": 132.71875, + "epoch": 0.24023668639053256, + "grad_norm": 0.3637740883688526, + "kl": 0.02587890625, + "learning_rate": 7.597633136094674e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 406 + }, + { + "completion_length": 128.5, + "epoch": 0.2408284023668639, + "grad_norm": 0.4484159338997197, + "kl": 0.0341796875, + "learning_rate": 7.591715976331361e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 407 + }, + { + "completion_length": 126.78125, + "epoch": 0.24142011834319527, + "grad_norm": 0.4583410495550597, + "kl": 0.033203125, + "learning_rate": 7.585798816568048e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 408 + }, + { + "completion_length": 127.0859375, + "epoch": 0.24201183431952664, + "grad_norm": 0.6293150515946456, + "kl": 0.03515625, + "learning_rate": 7.579881656804733e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 409 + }, + { + "completion_length": 134.8671875, + "epoch": 0.24260355029585798, + "grad_norm": 0.7278577378567606, + "kl": 0.0252685546875, + "learning_rate": 7.57396449704142e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 410 + }, + { + "completion_length": 121.140625, + "epoch": 0.24319526627218935, + "grad_norm": 0.48774877636284697, + "kl": 0.03173828125, + "learning_rate": 7.568047337278106e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 411 + }, + { + "completion_length": 125.4765625, + "epoch": 0.24378698224852072, + "grad_norm": 0.5557399616773951, + "kl": 0.0306396484375, + "learning_rate": 7.562130177514793e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 412 + }, + { + "completion_length": 130.8671875, + "epoch": 0.24437869822485206, + "grad_norm": 0.46131829540733377, + "kl": 0.0306396484375, + "learning_rate": 7.556213017751479e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 413 + }, + { + "completion_length": 127.375, + "epoch": 0.24497041420118343, + "grad_norm": 0.7118538684392436, + "kl": 0.0419921875, + "learning_rate": 7.550295857988165e-07, + "loss": 0.0017, + "reward": 0.125, + "reward_std": 0.11662659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 414 + }, + { + "completion_length": 133.6953125, + "epoch": 0.2455621301775148, + "grad_norm": 1.1978177406497186, + "kl": 0.06201171875, + "learning_rate": 7.544378698224852e-07, + "loss": 0.0025, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 415 + }, + { + "completion_length": 128.234375, + "epoch": 0.24615384615384617, + "grad_norm": 0.61214399235321, + "kl": 0.028564453125, + "learning_rate": 7.538461538461538e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 416 + }, + { + "completion_length": 122.5546875, + "epoch": 0.2467455621301775, + "grad_norm": 0.7191633908830194, + "kl": 0.03466796875, + "learning_rate": 7.532544378698224e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 417 + }, + { + "completion_length": 122.2421875, + "epoch": 0.24733727810650888, + "grad_norm": 0.6247886255968453, + "kl": 0.033203125, + "learning_rate": 7.526627218934911e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 418 + }, + { + "completion_length": 125.9375, + "epoch": 0.24792899408284025, + "grad_norm": 0.473866674915362, + "kl": 0.0361328125, + "learning_rate": 7.520710059171598e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 419 + }, + { + "completion_length": 132.0, + "epoch": 0.2485207100591716, + "grad_norm": 0.5760962191568125, + "kl": 0.0289306640625, + "learning_rate": 7.514792899408284e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 420 + }, + { + "completion_length": 133.015625, + "epoch": 0.24911242603550296, + "grad_norm": 0.41984589511136283, + "kl": 0.026611328125, + "learning_rate": 7.508875739644969e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 421 + }, + { + "completion_length": 138.1796875, + "epoch": 0.24970414201183433, + "grad_norm": 0.6440049230967643, + "kl": 0.033447265625, + "learning_rate": 7.502958579881656e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 422 + }, + { + "completion_length": 127.7109375, + "epoch": 0.25029585798816567, + "grad_norm": 0.29619790596535944, + "kl": 0.025146484375, + "learning_rate": 7.497041420118343e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 423 + }, + { + "completion_length": 127.7421875, + "epoch": 0.25088757396449707, + "grad_norm": 1.0020176000789291, + "kl": 0.02880859375, + "learning_rate": 7.49112426035503e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 424 + }, + { + "completion_length": 118.6171875, + "epoch": 0.2514792899408284, + "grad_norm": 0.8591400544025722, + "kl": 0.04345703125, + "learning_rate": 7.485207100591716e-07, + "loss": 0.0017, + "reward": 0.1953125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 425 + }, + { + "completion_length": 122.3828125, + "epoch": 0.25207100591715975, + "grad_norm": 0.2921964147413966, + "kl": 0.03515625, + "learning_rate": 7.479289940828401e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 426 + }, + { + "completion_length": 122.7578125, + "epoch": 0.25266272189349115, + "grad_norm": 3.820610394335818, + "kl": 0.03369140625, + "learning_rate": 7.473372781065088e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 427 + }, + { + "completion_length": 130.578125, + "epoch": 0.2532544378698225, + "grad_norm": 0.532827774364727, + "kl": 0.03662109375, + "learning_rate": 7.467455621301775e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 428 + }, + { + "completion_length": 117.7890625, + "epoch": 0.25384615384615383, + "grad_norm": 0.6954216281615448, + "kl": 0.031494140625, + "learning_rate": 7.461538461538461e-07, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 429 + }, + { + "completion_length": 122.2421875, + "epoch": 0.25443786982248523, + "grad_norm": 0.09853360498127872, + "kl": 0.0235595703125, + "learning_rate": 7.455621301775148e-07, + "loss": 0.0009, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 430 + }, + { + "completion_length": 118.6015625, + "epoch": 0.25502958579881657, + "grad_norm": 0.4173822605027055, + "kl": 0.0283203125, + "learning_rate": 7.449704142011835e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 431 + }, + { + "completion_length": 121.390625, + "epoch": 0.2556213017751479, + "grad_norm": 0.6304969463076443, + "kl": 0.028564453125, + "learning_rate": 7.44378698224852e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 432 + }, + { + "completion_length": 120.9375, + "epoch": 0.2562130177514793, + "grad_norm": 0.5672230039695126, + "kl": 0.03125, + "learning_rate": 7.437869822485206e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 433 + }, + { + "completion_length": 122.453125, + "epoch": 0.25680473372781065, + "grad_norm": 0.40999043335675495, + "kl": 0.032958984375, + "learning_rate": 7.431952662721893e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 434 + }, + { + "completion_length": 117.796875, + "epoch": 0.257396449704142, + "grad_norm": 0.6385749833245166, + "kl": 0.03466796875, + "learning_rate": 7.42603550295858e-07, + "loss": 0.0014, + "reward": 0.046875, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 435 + }, + { + "completion_length": 123.2109375, + "epoch": 0.2579881656804734, + "grad_norm": 0.7006341908776089, + "kl": 0.0299072265625, + "learning_rate": 7.420118343195267e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 436 + }, + { + "completion_length": 121.0078125, + "epoch": 0.25857988165680473, + "grad_norm": 0.07384197620339486, + "kl": 0.033203125, + "learning_rate": 7.414201183431951e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 437 + }, + { + "completion_length": 122.8828125, + "epoch": 0.2591715976331361, + "grad_norm": 0.5878768705752176, + "kl": 0.050048828125, + "learning_rate": 7.408284023668638e-07, + "loss": 0.002, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 438 + }, + { + "completion_length": 120.34375, + "epoch": 0.25976331360946747, + "grad_norm": 0.7619718313037258, + "kl": 0.0361328125, + "learning_rate": 7.402366863905325e-07, + "loss": 0.0014, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 439 + }, + { + "completion_length": 115.59375, + "epoch": 0.2603550295857988, + "grad_norm": 0.384814920491973, + "kl": 0.028564453125, + "learning_rate": 7.396449704142012e-07, + "loss": 0.0011, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 440 + }, + { + "completion_length": 129.9140625, + "epoch": 0.26094674556213016, + "grad_norm": 0.07107674976985566, + "kl": 0.0301513671875, + "learning_rate": 7.390532544378699e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 441 + }, + { + "completion_length": 124.7578125, + "epoch": 0.26153846153846155, + "grad_norm": 0.5394200497179126, + "kl": 0.032470703125, + "learning_rate": 7.384615384615384e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 442 + }, + { + "completion_length": 119.203125, + "epoch": 0.2621301775147929, + "grad_norm": 0.7150132849573293, + "kl": 0.035888671875, + "learning_rate": 7.37869822485207e-07, + "loss": 0.0014, + "reward": 0.21875, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.21875, + "step": 443 + }, + { + "completion_length": 122.2109375, + "epoch": 0.26272189349112424, + "grad_norm": 0.5602240325154372, + "kl": 0.0274658203125, + "learning_rate": 7.372781065088757e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 444 + }, + { + "completion_length": 121.421875, + "epoch": 0.26331360946745563, + "grad_norm": 0.4058269128817564, + "kl": 0.028564453125, + "learning_rate": 7.366863905325444e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 445 + }, + { + "completion_length": 116.03125, + "epoch": 0.263905325443787, + "grad_norm": 0.6728286999984049, + "kl": 0.0203857421875, + "learning_rate": 7.36094674556213e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 446 + }, + { + "completion_length": 115.4609375, + "epoch": 0.2644970414201183, + "grad_norm": 0.5439280356741116, + "kl": 0.02734375, + "learning_rate": 7.355029585798816e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 447 + }, + { + "completion_length": 111.921875, + "epoch": 0.2650887573964497, + "grad_norm": 0.5196087500294967, + "kl": 0.0286865234375, + "learning_rate": 7.349112426035503e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 448 + }, + { + "completion_length": 116.015625, + "epoch": 0.26568047337278106, + "grad_norm": 0.08014466425335293, + "kl": 0.0228271484375, + "learning_rate": 7.343195266272189e-07, + "loss": 0.0009, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 449 + }, + { + "completion_length": 109.0546875, + "epoch": 0.26627218934911245, + "grad_norm": 0.08179775382275328, + "kl": 0.0211181640625, + "learning_rate": 7.337278106508875e-07, + "loss": 0.0008, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 450 + }, + { + "completion_length": 110.5546875, + "epoch": 0.2668639053254438, + "grad_norm": 0.26608473166257085, + "kl": 0.025146484375, + "learning_rate": 7.331360946745562e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 451 + }, + { + "completion_length": 116.359375, + "epoch": 0.26745562130177514, + "grad_norm": 0.4559287394488598, + "kl": 0.0257568359375, + "learning_rate": 7.325443786982248e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 452 + }, + { + "completion_length": 120.8046875, + "epoch": 0.26804733727810653, + "grad_norm": 0.3594350144506889, + "kl": 0.0263671875, + "learning_rate": 7.319526627218935e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 453 + }, + { + "completion_length": 113.75, + "epoch": 0.2686390532544379, + "grad_norm": 0.7973406825741631, + "kl": 0.0244140625, + "learning_rate": 7.313609467455621e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 454 + }, + { + "completion_length": 117.5625, + "epoch": 0.2692307692307692, + "grad_norm": 0.4844323394934278, + "kl": 0.02197265625, + "learning_rate": 7.307692307692307e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 455 + }, + { + "completion_length": 121.671875, + "epoch": 0.2698224852071006, + "grad_norm": 0.7255631288144484, + "kl": 0.0277099609375, + "learning_rate": 7.301775147928994e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 456 + }, + { + "completion_length": 114.5546875, + "epoch": 0.27041420118343196, + "grad_norm": 0.6495633501406947, + "kl": 0.023681640625, + "learning_rate": 7.29585798816568e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 457 + }, + { + "completion_length": 117.9921875, + "epoch": 0.2710059171597633, + "grad_norm": 0.2886928753268644, + "kl": 0.03173828125, + "learning_rate": 7.289940828402366e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 458 + }, + { + "completion_length": 122.1640625, + "epoch": 0.2715976331360947, + "grad_norm": 1.007391264652057, + "kl": 0.022216796875, + "learning_rate": 7.284023668639053e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 459 + }, + { + "completion_length": 113.375, + "epoch": 0.27218934911242604, + "grad_norm": 0.5179004141392066, + "kl": 0.023681640625, + "learning_rate": 7.27810650887574e-07, + "loss": 0.0009, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 460 + }, + { + "completion_length": 123.0078125, + "epoch": 0.2727810650887574, + "grad_norm": 0.7728699133899016, + "kl": 0.0308837890625, + "learning_rate": 7.272189349112426e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 461 + }, + { + "completion_length": 112.0, + "epoch": 0.2733727810650888, + "grad_norm": 0.482725247218076, + "kl": 0.0263671875, + "learning_rate": 7.266272189349111e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 462 + }, + { + "completion_length": 123.265625, + "epoch": 0.2739644970414201, + "grad_norm": 0.48881367755181576, + "kl": 0.0306396484375, + "learning_rate": 7.260355029585798e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 463 + }, + { + "completion_length": 130.578125, + "epoch": 0.27455621301775146, + "grad_norm": 0.5658398711022044, + "kl": 0.024658203125, + "learning_rate": 7.254437869822485e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 464 + }, + { + "completion_length": 118.5, + "epoch": 0.27514792899408286, + "grad_norm": 0.5682943148350822, + "kl": 0.0260009765625, + "learning_rate": 7.248520710059172e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 465 + }, + { + "completion_length": 117.484375, + "epoch": 0.2757396449704142, + "grad_norm": 0.6272341956531508, + "kl": 0.0205078125, + "learning_rate": 7.242603550295857e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 466 + }, + { + "completion_length": 127.3125, + "epoch": 0.27633136094674554, + "grad_norm": 0.7374334361245112, + "kl": 0.0201416015625, + "learning_rate": 7.236686390532543e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 467 + }, + { + "completion_length": 125.828125, + "epoch": 0.27692307692307694, + "grad_norm": 0.5402390144354512, + "kl": 0.02001953125, + "learning_rate": 7.23076923076923e-07, + "loss": 0.0008, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 468 + }, + { + "completion_length": 120.1640625, + "epoch": 0.2775147928994083, + "grad_norm": 0.6015043357901544, + "kl": 0.023193359375, + "learning_rate": 7.224852071005917e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 469 + }, + { + "completion_length": 121.625, + "epoch": 0.2781065088757396, + "grad_norm": 0.6443240986431388, + "kl": 0.028076171875, + "learning_rate": 7.218934911242604e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 470 + }, + { + "completion_length": 116.9765625, + "epoch": 0.278698224852071, + "grad_norm": 0.4695242961534213, + "kl": 0.018798828125, + "learning_rate": 7.21301775147929e-07, + "loss": 0.0007, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 471 + }, + { + "completion_length": 118.4609375, + "epoch": 0.27928994082840236, + "grad_norm": 0.7020946611049157, + "kl": 0.02587890625, + "learning_rate": 7.207100591715975e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 472 + }, + { + "completion_length": 131.0625, + "epoch": 0.2798816568047337, + "grad_norm": 0.4466518083359742, + "kl": 0.024658203125, + "learning_rate": 7.201183431952662e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 473 + }, + { + "completion_length": 112.609375, + "epoch": 0.2804733727810651, + "grad_norm": 0.3225148279740824, + "kl": 0.024169921875, + "learning_rate": 7.195266272189349e-07, + "loss": 0.001, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 474 + }, + { + "completion_length": 120.859375, + "epoch": 0.28106508875739644, + "grad_norm": 0.5006700041114581, + "kl": 0.021484375, + "learning_rate": 7.189349112426035e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 475 + }, + { + "completion_length": 129.5390625, + "epoch": 0.2816568047337278, + "grad_norm": 0.6105923522633407, + "kl": 0.0289306640625, + "learning_rate": 7.183431952662722e-07, + "loss": 0.0012, + "reward": 0.03125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 476 + }, + { + "completion_length": 115.609375, + "epoch": 0.2822485207100592, + "grad_norm": 0.6386946861470317, + "kl": 0.023193359375, + "learning_rate": 7.177514792899409e-07, + "loss": 0.0009, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 477 + }, + { + "completion_length": 121.7890625, + "epoch": 0.2828402366863905, + "grad_norm": 1.648192276127079, + "kl": 0.0252685546875, + "learning_rate": 7.171597633136094e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 478 + }, + { + "completion_length": 117.1015625, + "epoch": 0.2834319526627219, + "grad_norm": 0.5719877983395144, + "kl": 0.0269775390625, + "learning_rate": 7.16568047337278e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 479 + }, + { + "completion_length": 126.0546875, + "epoch": 0.28402366863905326, + "grad_norm": 0.6272133337822063, + "kl": 0.042724609375, + "learning_rate": 7.159763313609467e-07, + "loss": 0.0017, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 480 + }, + { + "completion_length": 122.359375, + "epoch": 0.2846153846153846, + "grad_norm": 0.7566153886921407, + "kl": 0.02978515625, + "learning_rate": 7.153846153846154e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 481 + }, + { + "completion_length": 121.8046875, + "epoch": 0.285207100591716, + "grad_norm": 1.0688041807868978, + "kl": 0.0291748046875, + "learning_rate": 7.147928994082841e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 482 + }, + { + "completion_length": 127.1796875, + "epoch": 0.28579881656804734, + "grad_norm": 0.48583347576836317, + "kl": 0.027099609375, + "learning_rate": 7.142011834319526e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 483 + }, + { + "completion_length": 114.4921875, + "epoch": 0.2863905325443787, + "grad_norm": 0.5904731903471492, + "kl": 0.0277099609375, + "learning_rate": 7.136094674556212e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 484 + }, + { + "completion_length": 117.9609375, + "epoch": 0.2869822485207101, + "grad_norm": 0.7964192755143658, + "kl": 0.0228271484375, + "learning_rate": 7.130177514792899e-07, + "loss": 0.0009, + "reward": 0.140625, + "reward_std": 0.09858438372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 485 + }, + { + "completion_length": 134.8046875, + "epoch": 0.2875739644970414, + "grad_norm": 1.3216372783056993, + "kl": 0.0301513671875, + "learning_rate": 7.124260355029586e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 486 + }, + { + "completion_length": 127.484375, + "epoch": 0.28816568047337277, + "grad_norm": 0.31521692167435095, + "kl": 0.0211181640625, + "learning_rate": 7.118343195266272e-07, + "loss": 0.0008, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 487 + }, + { + "completion_length": 126.0078125, + "epoch": 0.28875739644970416, + "grad_norm": 0.619985899572795, + "kl": 0.024658203125, + "learning_rate": 7.112426035502958e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 488 + }, + { + "completion_length": 116.1640625, + "epoch": 0.2893491124260355, + "grad_norm": 0.13645444618886582, + "kl": 0.027587890625, + "learning_rate": 7.106508875739645e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 489 + }, + { + "completion_length": 131.546875, + "epoch": 0.28994082840236685, + "grad_norm": 0.5603652499238605, + "kl": 0.02587890625, + "learning_rate": 7.100591715976331e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 490 + }, + { + "completion_length": 127.6953125, + "epoch": 0.29053254437869824, + "grad_norm": 0.3978479290907902, + "kl": 0.025146484375, + "learning_rate": 7.094674556213017e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 491 + }, + { + "completion_length": 121.9765625, + "epoch": 0.2911242603550296, + "grad_norm": 0.8126923665650364, + "kl": 0.028564453125, + "learning_rate": 7.088757396449704e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.09858438372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 492 + }, + { + "completion_length": 128.8671875, + "epoch": 0.2917159763313609, + "grad_norm": 0.06791389833504351, + "kl": 0.0205078125, + "learning_rate": 7.08284023668639e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 493 + }, + { + "completion_length": 127.5625, + "epoch": 0.2923076923076923, + "grad_norm": 0.767320321215326, + "kl": 0.036865234375, + "learning_rate": 7.076923076923077e-07, + "loss": 0.0015, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 494 + }, + { + "completion_length": 125.2578125, + "epoch": 0.29289940828402367, + "grad_norm": 0.5450823397641261, + "kl": 0.0235595703125, + "learning_rate": 7.071005917159762e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 495 + }, + { + "completion_length": 123.1953125, + "epoch": 0.293491124260355, + "grad_norm": 0.4613798099869595, + "kl": 0.034912109375, + "learning_rate": 7.065088757396449e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 496 + }, + { + "completion_length": 129.765625, + "epoch": 0.2940828402366864, + "grad_norm": 0.2996591181102951, + "kl": 0.01953125, + "learning_rate": 7.059171597633136e-07, + "loss": 0.0008, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 497 + }, + { + "completion_length": 121.171875, + "epoch": 0.29467455621301775, + "grad_norm": 0.3101606596635717, + "kl": 0.028564453125, + "learning_rate": 7.053254437869822e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 498 + }, + { + "completion_length": 120.71875, + "epoch": 0.2952662721893491, + "grad_norm": 0.7022945336448755, + "kl": 0.0279541015625, + "learning_rate": 7.047337278106509e-07, + "loss": 0.0011, + "reward": 0.1328125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 499 + }, + { + "completion_length": 114.9921875, + "epoch": 0.2958579881656805, + "grad_norm": 0.4518756162044563, + "kl": 0.018310546875, + "learning_rate": 7.041420118343195e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 500 + }, + { + "completion_length": 120.859375, + "epoch": 0.2964497041420118, + "grad_norm": 0.4950418731922019, + "kl": 0.019775390625, + "learning_rate": 7.035502958579881e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 501 + }, + { + "completion_length": 122.796875, + "epoch": 0.29704142011834317, + "grad_norm": 0.5193517317772786, + "kl": 0.0240478515625, + "learning_rate": 7.029585798816568e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 502 + }, + { + "completion_length": 114.765625, + "epoch": 0.29763313609467457, + "grad_norm": 0.43238320436917427, + "kl": 0.0296630859375, + "learning_rate": 7.023668639053254e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 503 + }, + { + "completion_length": 114.1015625, + "epoch": 0.2982248520710059, + "grad_norm": 0.7609245055775616, + "kl": 0.0283203125, + "learning_rate": 7.01775147928994e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 504 + }, + { + "completion_length": 114.625, + "epoch": 0.2988165680473373, + "grad_norm": 0.6927458019356328, + "kl": 0.02880859375, + "learning_rate": 7.011834319526627e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 505 + }, + { + "completion_length": 117.9765625, + "epoch": 0.29940828402366865, + "grad_norm": 0.6800632617601882, + "kl": 0.022216796875, + "learning_rate": 7.005917159763314e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 506 + }, + { + "completion_length": 106.4921875, + "epoch": 0.3, + "grad_norm": 0.058258934869385964, + "kl": 0.01953125, + "learning_rate": 7e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 507 + }, + { + "completion_length": 118.65625, + "epoch": 0.3005917159763314, + "grad_norm": 0.860555080761317, + "kl": 0.0240478515625, + "learning_rate": 6.994082840236685e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 508 + }, + { + "completion_length": 119.390625, + "epoch": 0.30118343195266273, + "grad_norm": 0.614787976563766, + "kl": 0.0224609375, + "learning_rate": 6.988165680473372e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 509 + }, + { + "completion_length": 111.5234375, + "epoch": 0.30177514792899407, + "grad_norm": 0.7340367461363745, + "kl": 0.033935546875, + "learning_rate": 6.982248520710059e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 510 + }, + { + "completion_length": 110.2734375, + "epoch": 0.30236686390532547, + "grad_norm": 0.5816223914864312, + "kl": 0.024658203125, + "learning_rate": 6.976331360946746e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 511 + }, + { + "completion_length": 111.3828125, + "epoch": 0.3029585798816568, + "grad_norm": 0.9953313354967308, + "kl": 0.034912109375, + "learning_rate": 6.970414201183432e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.08295938372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 512 + }, + { + "completion_length": 113.671875, + "epoch": 0.30355029585798815, + "grad_norm": 0.5549188139783847, + "kl": 0.0224609375, + "learning_rate": 6.964497041420117e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 513 + }, + { + "completion_length": 122.359375, + "epoch": 0.30414201183431955, + "grad_norm": 0.4130263972141377, + "kl": 0.02392578125, + "learning_rate": 6.958579881656804e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 514 + }, + { + "completion_length": 109.203125, + "epoch": 0.3047337278106509, + "grad_norm": 0.3143091795894772, + "kl": 0.0269775390625, + "learning_rate": 6.952662721893491e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 515 + }, + { + "completion_length": 115.0625, + "epoch": 0.30532544378698223, + "grad_norm": 0.4233803282071342, + "kl": 0.03271484375, + "learning_rate": 6.946745562130177e-07, + "loss": 0.0013, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 516 + }, + { + "completion_length": 119.96875, + "epoch": 0.30591715976331363, + "grad_norm": 0.7117266316810598, + "kl": 0.029541015625, + "learning_rate": 6.940828402366864e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 517 + }, + { + "completion_length": 120.921875, + "epoch": 0.30650887573964497, + "grad_norm": 0.6093708666026807, + "kl": 0.0308837890625, + "learning_rate": 6.93491124260355e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 518 + }, + { + "completion_length": 122.546875, + "epoch": 0.3071005917159763, + "grad_norm": 0.5403713932585138, + "kl": 0.030029296875, + "learning_rate": 6.928994082840236e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 519 + }, + { + "completion_length": 117.28125, + "epoch": 0.3076923076923077, + "grad_norm": 0.42848184604435463, + "kl": 0.0274658203125, + "learning_rate": 6.923076923076922e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 520 + }, + { + "completion_length": 119.984375, + "epoch": 0.30828402366863905, + "grad_norm": 1.1089609903178244, + "kl": 0.033447265625, + "learning_rate": 6.917159763313609e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 521 + }, + { + "completion_length": 108.5625, + "epoch": 0.3088757396449704, + "grad_norm": 0.3625488662494576, + "kl": 0.02734375, + "learning_rate": 6.911242603550296e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 522 + }, + { + "completion_length": 128.4375, + "epoch": 0.3094674556213018, + "grad_norm": 0.6740766488615482, + "kl": 0.03173828125, + "learning_rate": 6.905325443786983e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 523 + }, + { + "completion_length": 119.7734375, + "epoch": 0.31005917159763313, + "grad_norm": 0.6485011785819526, + "kl": 0.0311279296875, + "learning_rate": 6.899408284023667e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 524 + }, + { + "completion_length": 116.8984375, + "epoch": 0.3106508875739645, + "grad_norm": 0.5660052022884334, + "kl": 0.03369140625, + "learning_rate": 6.893491124260354e-07, + "loss": 0.0013, + "reward": 0.171875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 525 + }, + { + "completion_length": 130.8828125, + "epoch": 0.31124260355029587, + "grad_norm": 0.30588492509831444, + "kl": 0.0250244140625, + "learning_rate": 6.887573964497041e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 526 + }, + { + "completion_length": 130.09375, + "epoch": 0.3118343195266272, + "grad_norm": 0.6392787667534353, + "kl": 0.0245361328125, + "learning_rate": 6.881656804733728e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 527 + }, + { + "completion_length": 129.375, + "epoch": 0.31242603550295855, + "grad_norm": 0.5529647459100306, + "kl": 0.0263671875, + "learning_rate": 6.875739644970415e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 528 + }, + { + "completion_length": 136.9375, + "epoch": 0.31301775147928995, + "grad_norm": 0.39534987127586235, + "kl": 0.0311279296875, + "learning_rate": 6.8698224852071e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 529 + }, + { + "completion_length": 116.3671875, + "epoch": 0.3136094674556213, + "grad_norm": 0.688722492681135, + "kl": 0.03076171875, + "learning_rate": 6.863905325443786e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 530 + }, + { + "completion_length": 120.6640625, + "epoch": 0.31420118343195264, + "grad_norm": 0.6198836187397745, + "kl": 0.033203125, + "learning_rate": 6.857988165680473e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 531 + }, + { + "completion_length": 120.28125, + "epoch": 0.31479289940828403, + "grad_norm": 0.42642347786039914, + "kl": 0.02294921875, + "learning_rate": 6.85207100591716e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 532 + }, + { + "completion_length": 117.2890625, + "epoch": 0.3153846153846154, + "grad_norm": 0.5846748465369991, + "kl": 0.032958984375, + "learning_rate": 6.846153846153846e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 533 + }, + { + "completion_length": 119.265625, + "epoch": 0.31597633136094677, + "grad_norm": 0.8000443767564643, + "kl": 0.0419921875, + "learning_rate": 6.840236686390532e-07, + "loss": 0.0017, + "reward": 0.109375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 534 + }, + { + "completion_length": 110.8046875, + "epoch": 0.3165680473372781, + "grad_norm": 0.6185785682939023, + "kl": 0.036376953125, + "learning_rate": 6.834319526627219e-07, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 535 + }, + { + "completion_length": 115.9453125, + "epoch": 0.31715976331360946, + "grad_norm": 0.8404585572627266, + "kl": 0.03369140625, + "learning_rate": 6.828402366863905e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 536 + }, + { + "completion_length": 117.2265625, + "epoch": 0.31775147928994085, + "grad_norm": 0.5163175181225461, + "kl": 0.0291748046875, + "learning_rate": 6.822485207100591e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 537 + }, + { + "completion_length": 120.0703125, + "epoch": 0.3183431952662722, + "grad_norm": 0.3813109100768173, + "kl": 0.021484375, + "learning_rate": 6.816568047337278e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 538 + }, + { + "completion_length": 120.8671875, + "epoch": 0.31893491124260354, + "grad_norm": 0.24654120277934774, + "kl": 0.0311279296875, + "learning_rate": 6.810650887573964e-07, + "loss": 0.0012, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 539 + }, + { + "completion_length": 112.25, + "epoch": 0.31952662721893493, + "grad_norm": 0.42713544785369184, + "kl": 0.033203125, + "learning_rate": 6.804733727810651e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 540 + }, + { + "completion_length": 110.8359375, + "epoch": 0.3201183431952663, + "grad_norm": 0.8180705652327475, + "kl": 0.06494140625, + "learning_rate": 6.798816568047337e-07, + "loss": 0.0026, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 541 + }, + { + "completion_length": 119.7421875, + "epoch": 0.3207100591715976, + "grad_norm": 0.5184760853722311, + "kl": 0.04296875, + "learning_rate": 6.792899408284023e-07, + "loss": 0.0017, + "reward": 0.109375, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 542 + }, + { + "completion_length": 117.0546875, + "epoch": 0.321301775147929, + "grad_norm": 1.1812753095596908, + "kl": 0.04052734375, + "learning_rate": 6.78698224852071e-07, + "loss": 0.0016, + "reward": 0.1640625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 543 + }, + { + "completion_length": 123.0078125, + "epoch": 0.32189349112426036, + "grad_norm": 0.6363792622441924, + "kl": 0.039306640625, + "learning_rate": 6.781065088757396e-07, + "loss": 0.0016, + "reward": 0.1484375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 544 + }, + { + "completion_length": 122.609375, + "epoch": 0.3224852071005917, + "grad_norm": 0.5375125375773516, + "kl": 0.03759765625, + "learning_rate": 6.775147928994082e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 545 + }, + { + "completion_length": 112.3984375, + "epoch": 0.3230769230769231, + "grad_norm": 0.44115338493134365, + "kl": 0.0361328125, + "learning_rate": 6.769230769230769e-07, + "loss": 0.0014, + "reward": 0.171875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 546 + }, + { + "completion_length": 113.9296875, + "epoch": 0.32366863905325444, + "grad_norm": 0.431477235952646, + "kl": 0.027099609375, + "learning_rate": 6.763313609467455e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 547 + }, + { + "completion_length": 111.59375, + "epoch": 0.3242603550295858, + "grad_norm": 1.274414506170159, + "kl": 0.05517578125, + "learning_rate": 6.757396449704142e-07, + "loss": 0.0022, + "reward": 0.1640625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 548 + }, + { + "completion_length": 114.859375, + "epoch": 0.3248520710059172, + "grad_norm": 0.5891692839663576, + "kl": 0.038330078125, + "learning_rate": 6.751479289940827e-07, + "loss": 0.0015, + "reward": 0.1640625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 549 + }, + { + "completion_length": 118.6640625, + "epoch": 0.3254437869822485, + "grad_norm": 0.5027550064857566, + "kl": 0.0302734375, + "learning_rate": 6.745562130177514e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 550 + }, + { + "completion_length": 114.8828125, + "epoch": 0.32603550295857986, + "grad_norm": 0.0791413619016759, + "kl": 0.027099609375, + "learning_rate": 6.739644970414201e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 551 + }, + { + "completion_length": 118.421875, + "epoch": 0.32662721893491126, + "grad_norm": 0.5654134613565681, + "kl": 0.03662109375, + "learning_rate": 6.733727810650888e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 552 + }, + { + "completion_length": 124.796875, + "epoch": 0.3272189349112426, + "grad_norm": 0.6023154616909698, + "kl": 0.032958984375, + "learning_rate": 6.727810650887573e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 553 + }, + { + "completion_length": 119.125, + "epoch": 0.32781065088757394, + "grad_norm": 0.6310823231090252, + "kl": 0.031494140625, + "learning_rate": 6.72189349112426e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 554 + }, + { + "completion_length": 126.3671875, + "epoch": 0.32840236686390534, + "grad_norm": 0.593127992078192, + "kl": 0.03173828125, + "learning_rate": 6.715976331360946e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.07216878235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 555 + }, + { + "completion_length": 122.8359375, + "epoch": 0.3289940828402367, + "grad_norm": 0.304003504178165, + "kl": 0.0233154296875, + "learning_rate": 6.710059171597633e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 556 + }, + { + "completion_length": 115.1328125, + "epoch": 0.329585798816568, + "grad_norm": 0.9147868705931621, + "kl": 0.0341796875, + "learning_rate": 6.70414201183432e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 557 + }, + { + "completion_length": 116.7890625, + "epoch": 0.3301775147928994, + "grad_norm": 0.46540124830276375, + "kl": 0.0361328125, + "learning_rate": 6.698224852071006e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 558 + }, + { + "completion_length": 127.015625, + "epoch": 0.33076923076923076, + "grad_norm": 0.587281083380056, + "kl": 0.031982421875, + "learning_rate": 6.692307692307692e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 559 + }, + { + "completion_length": 119.109375, + "epoch": 0.33136094674556216, + "grad_norm": 0.9690844771843253, + "kl": 0.04248046875, + "learning_rate": 6.686390532544378e-07, + "loss": 0.0017, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 560 + }, + { + "completion_length": 120.1953125, + "epoch": 0.3319526627218935, + "grad_norm": 0.07933831432524714, + "kl": 0.025146484375, + "learning_rate": 6.680473372781065e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 561 + }, + { + "completion_length": 119.828125, + "epoch": 0.33254437869822484, + "grad_norm": 0.7612338869254905, + "kl": 0.03076171875, + "learning_rate": 6.674556213017751e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 562 + }, + { + "completion_length": 123.1484375, + "epoch": 0.33313609467455624, + "grad_norm": 0.39728747257291464, + "kl": 0.0260009765625, + "learning_rate": 6.668639053254438e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 563 + }, + { + "completion_length": 123.15625, + "epoch": 0.3337278106508876, + "grad_norm": 0.8062269781086536, + "kl": 0.037841796875, + "learning_rate": 6.662721893491125e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 564 + }, + { + "completion_length": 123.59375, + "epoch": 0.3343195266272189, + "grad_norm": 0.5763953877174803, + "kl": 0.03369140625, + "learning_rate": 6.65680473372781e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 565 + }, + { + "completion_length": 131.828125, + "epoch": 0.3349112426035503, + "grad_norm": 0.393614113129736, + "kl": 0.03759765625, + "learning_rate": 6.650887573964496e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 566 + }, + { + "completion_length": 123.96875, + "epoch": 0.33550295857988166, + "grad_norm": 0.4407040309685585, + "kl": 0.029541015625, + "learning_rate": 6.644970414201183e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 567 + }, + { + "completion_length": 127.5546875, + "epoch": 0.336094674556213, + "grad_norm": 0.56468645098456, + "kl": 0.030029296875, + "learning_rate": 6.63905325443787e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 568 + }, + { + "completion_length": 126.9140625, + "epoch": 0.3366863905325444, + "grad_norm": 0.5882866758084621, + "kl": 0.0272216796875, + "learning_rate": 6.633136094674557e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 569 + }, + { + "completion_length": 123.296875, + "epoch": 0.33727810650887574, + "grad_norm": 0.6081971420529366, + "kl": 0.03662109375, + "learning_rate": 6.627218934911242e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 570 + }, + { + "completion_length": 117.3671875, + "epoch": 0.3378698224852071, + "grad_norm": 0.8417354310037387, + "kl": 0.04150390625, + "learning_rate": 6.621301775147928e-07, + "loss": 0.0017, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 571 + }, + { + "completion_length": 127.8359375, + "epoch": 0.3384615384615385, + "grad_norm": 0.15783687993209977, + "kl": 0.03759765625, + "learning_rate": 6.615384615384615e-07, + "loss": 0.0015, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 572 + }, + { + "completion_length": 129.7265625, + "epoch": 0.3390532544378698, + "grad_norm": 0.4333503911088854, + "kl": 0.036376953125, + "learning_rate": 6.609467455621302e-07, + "loss": 0.0015, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 573 + }, + { + "completion_length": 116.34375, + "epoch": 0.33964497041420116, + "grad_norm": 0.7353292506965768, + "kl": 0.029052734375, + "learning_rate": 6.603550295857988e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 574 + }, + { + "completion_length": 129.578125, + "epoch": 0.34023668639053256, + "grad_norm": 0.4902320175635443, + "kl": 0.031494140625, + "learning_rate": 6.597633136094674e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 575 + }, + { + "completion_length": 128.265625, + "epoch": 0.3408284023668639, + "grad_norm": 0.45544492719636687, + "kl": 0.0303955078125, + "learning_rate": 6.59171597633136e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 576 + }, + { + "completion_length": 133.546875, + "epoch": 0.34142011834319524, + "grad_norm": 0.5423247750027944, + "kl": 0.027099609375, + "learning_rate": 6.585798816568047e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 577 + }, + { + "completion_length": 121.1875, + "epoch": 0.34201183431952664, + "grad_norm": 0.22638640369156116, + "kl": 0.0255126953125, + "learning_rate": 6.579881656804733e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 578 + }, + { + "completion_length": 122.3515625, + "epoch": 0.342603550295858, + "grad_norm": 1.4151597519286592, + "kl": 0.04052734375, + "learning_rate": 6.57396449704142e-07, + "loss": 0.0016, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 579 + }, + { + "completion_length": 121.515625, + "epoch": 0.3431952662721893, + "grad_norm": 0.5847861576233347, + "kl": 0.0272216796875, + "learning_rate": 6.568047337278106e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 580 + }, + { + "completion_length": 129.46875, + "epoch": 0.3437869822485207, + "grad_norm": 0.5734767273739593, + "kl": 0.0255126953125, + "learning_rate": 6.562130177514793e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 581 + }, + { + "completion_length": 123.609375, + "epoch": 0.34437869822485206, + "grad_norm": 0.6099517729064499, + "kl": 0.038330078125, + "learning_rate": 6.556213017751478e-07, + "loss": 0.0015, + "reward": 0.2109375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2109375, + "step": 582 + }, + { + "completion_length": 120.7265625, + "epoch": 0.3449704142011834, + "grad_norm": 1.2055878907779924, + "kl": 0.03369140625, + "learning_rate": 6.550295857988165e-07, + "loss": 0.0014, + "reward": 0.1484375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 583 + }, + { + "completion_length": 127.4140625, + "epoch": 0.3455621301775148, + "grad_norm": 0.48385198501144333, + "kl": 0.0284423828125, + "learning_rate": 6.544378698224852e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 584 + }, + { + "completion_length": 122.125, + "epoch": 0.34615384615384615, + "grad_norm": 0.5657970752740376, + "kl": 0.032470703125, + "learning_rate": 6.538461538461538e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 585 + }, + { + "completion_length": 124.7265625, + "epoch": 0.3467455621301775, + "grad_norm": 0.5937366317732885, + "kl": 0.033935546875, + "learning_rate": 6.532544378698225e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 586 + }, + { + "completion_length": 121.7734375, + "epoch": 0.3473372781065089, + "grad_norm": 0.11778699221943291, + "kl": 0.024169921875, + "learning_rate": 6.526627218934911e-07, + "loss": 0.001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 587 + }, + { + "completion_length": 118.171875, + "epoch": 0.3479289940828402, + "grad_norm": 0.654642811344116, + "kl": 0.033203125, + "learning_rate": 6.520710059171597e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 588 + }, + { + "completion_length": 124.203125, + "epoch": 0.3485207100591716, + "grad_norm": 0.7068265201173398, + "kl": 0.0262451171875, + "learning_rate": 6.514792899408284e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 589 + }, + { + "completion_length": 137.1171875, + "epoch": 0.34911242603550297, + "grad_norm": 0.6100244562213681, + "kl": 0.03564453125, + "learning_rate": 6.50887573964497e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 590 + }, + { + "completion_length": 127.265625, + "epoch": 0.3497041420118343, + "grad_norm": 0.48808413884769025, + "kl": 0.030029296875, + "learning_rate": 6.502958579881656e-07, + "loss": 0.0012, + "reward": 0.1875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 591 + }, + { + "completion_length": 129.265625, + "epoch": 0.3502958579881657, + "grad_norm": 0.6031819747690056, + "kl": 0.0361328125, + "learning_rate": 6.497041420118343e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 592 + }, + { + "completion_length": 130.5859375, + "epoch": 0.35088757396449705, + "grad_norm": 0.38566076452869646, + "kl": 0.0294189453125, + "learning_rate": 6.49112426035503e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 593 + }, + { + "completion_length": 126.6796875, + "epoch": 0.3514792899408284, + "grad_norm": 0.5154114398997007, + "kl": 0.0250244140625, + "learning_rate": 6.485207100591716e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 594 + }, + { + "completion_length": 135.5234375, + "epoch": 0.3520710059171598, + "grad_norm": 0.5914120813052642, + "kl": 0.029296875, + "learning_rate": 6.479289940828401e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 595 + }, + { + "completion_length": 135.53125, + "epoch": 0.3526627218934911, + "grad_norm": 0.5390280937461622, + "kl": 0.0257568359375, + "learning_rate": 6.473372781065088e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 596 + }, + { + "completion_length": 138.7578125, + "epoch": 0.35325443786982247, + "grad_norm": 0.32850088146212875, + "kl": 0.026123046875, + "learning_rate": 6.467455621301775e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 597 + }, + { + "completion_length": 130.4921875, + "epoch": 0.35384615384615387, + "grad_norm": 0.6067485906499509, + "kl": 0.037353515625, + "learning_rate": 6.461538461538462e-07, + "loss": 0.0015, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 598 + }, + { + "completion_length": 131.0, + "epoch": 0.3544378698224852, + "grad_norm": 0.5618675185570801, + "kl": 0.0341796875, + "learning_rate": 6.455621301775148e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 599 + }, + { + "completion_length": 138.6875, + "epoch": 0.35502958579881655, + "grad_norm": 0.5061373316177682, + "kl": 0.042236328125, + "learning_rate": 6.449704142011834e-07, + "loss": 0.0017, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 600 + }, + { + "completion_length": 134.8515625, + "epoch": 0.35562130177514795, + "grad_norm": 0.47412727649904873, + "kl": 0.0302734375, + "learning_rate": 6.44378698224852e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 601 + }, + { + "completion_length": 122.890625, + "epoch": 0.3562130177514793, + "grad_norm": 0.660616694423821, + "kl": 0.034912109375, + "learning_rate": 6.437869822485207e-07, + "loss": 0.0014, + "reward": 0.21875, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.21875, + "step": 602 + }, + { + "completion_length": 118.40625, + "epoch": 0.35680473372781063, + "grad_norm": 0.2782276817761258, + "kl": 0.03955078125, + "learning_rate": 6.431952662721893e-07, + "loss": 0.0016, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 603 + }, + { + "completion_length": 131.625, + "epoch": 0.357396449704142, + "grad_norm": 0.834951601177143, + "kl": 0.02978515625, + "learning_rate": 6.42603550295858e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 604 + }, + { + "completion_length": 123.28125, + "epoch": 0.35798816568047337, + "grad_norm": 0.34777685853283524, + "kl": 0.03125, + "learning_rate": 6.420118343195266e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 605 + }, + { + "completion_length": 135.4453125, + "epoch": 0.3585798816568047, + "grad_norm": 0.6937862245829861, + "kl": 0.0245361328125, + "learning_rate": 6.414201183431952e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 606 + }, + { + "completion_length": 119.203125, + "epoch": 0.3591715976331361, + "grad_norm": 0.39180944932867895, + "kl": 0.035888671875, + "learning_rate": 6.408284023668638e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 607 + }, + { + "completion_length": 119.4375, + "epoch": 0.35976331360946745, + "grad_norm": 0.6452341308121187, + "kl": 0.0341796875, + "learning_rate": 6.402366863905325e-07, + "loss": 0.0014, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 608 + }, + { + "completion_length": 121.8984375, + "epoch": 0.3603550295857988, + "grad_norm": 0.7608484644177793, + "kl": 0.0257568359375, + "learning_rate": 6.396449704142012e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 609 + }, + { + "completion_length": 121.359375, + "epoch": 0.3609467455621302, + "grad_norm": 0.7691940241836015, + "kl": 0.031005859375, + "learning_rate": 6.390532544378699e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 610 + }, + { + "completion_length": 129.453125, + "epoch": 0.36153846153846153, + "grad_norm": 0.8218295981726239, + "kl": 0.033203125, + "learning_rate": 6.384615384615383e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 611 + }, + { + "completion_length": 125.9921875, + "epoch": 0.3621301775147929, + "grad_norm": 0.3960406265298925, + "kl": 0.031005859375, + "learning_rate": 6.37869822485207e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 612 + }, + { + "completion_length": 129.703125, + "epoch": 0.36272189349112427, + "grad_norm": 0.6576686935049824, + "kl": 0.03515625, + "learning_rate": 6.372781065088757e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 613 + }, + { + "completion_length": 122.3359375, + "epoch": 0.3633136094674556, + "grad_norm": 0.3428308631724517, + "kl": 0.036865234375, + "learning_rate": 6.366863905325444e-07, + "loss": 0.0015, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 614 + }, + { + "completion_length": 122.7421875, + "epoch": 0.363905325443787, + "grad_norm": 1.3114564623635383, + "kl": 0.02880859375, + "learning_rate": 6.360946745562131e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 615 + }, + { + "completion_length": 121.671875, + "epoch": 0.36449704142011835, + "grad_norm": 0.7900497284989573, + "kl": 0.027099609375, + "learning_rate": 6.355029585798816e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 616 + }, + { + "completion_length": 117.6484375, + "epoch": 0.3650887573964497, + "grad_norm": 0.29102044322735965, + "kl": 0.03125, + "learning_rate": 6.349112426035502e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 617 + }, + { + "completion_length": 123.1328125, + "epoch": 0.3656804733727811, + "grad_norm": 0.6151660982176845, + "kl": 0.032470703125, + "learning_rate": 6.343195266272189e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.08295938372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 618 + }, + { + "completion_length": 122.3671875, + "epoch": 0.36627218934911243, + "grad_norm": 0.6395295048834302, + "kl": 0.034423828125, + "learning_rate": 6.337278106508876e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 619 + }, + { + "completion_length": 120.3515625, + "epoch": 0.3668639053254438, + "grad_norm": 1.2152511260267107, + "kl": 0.02734375, + "learning_rate": 6.331360946745562e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.078125, + "step": 620 + }, + { + "completion_length": 121.484375, + "epoch": 0.36745562130177517, + "grad_norm": 0.07769795595738317, + "kl": 0.0235595703125, + "learning_rate": 6.325443786982248e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 621 + }, + { + "completion_length": 110.6015625, + "epoch": 0.3680473372781065, + "grad_norm": 0.5657270723116865, + "kl": 0.04248046875, + "learning_rate": 6.319526627218935e-07, + "loss": 0.0017, + "reward": 0.1640625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 622 + }, + { + "completion_length": 128.0625, + "epoch": 0.36863905325443785, + "grad_norm": 0.3666277268385193, + "kl": 0.0286865234375, + "learning_rate": 6.313609467455621e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 623 + }, + { + "completion_length": 120.6015625, + "epoch": 0.36923076923076925, + "grad_norm": 0.3850463192191957, + "kl": 0.030029296875, + "learning_rate": 6.307692307692307e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 624 + }, + { + "completion_length": 128.5859375, + "epoch": 0.3698224852071006, + "grad_norm": 0.44605663452606253, + "kl": 0.0245361328125, + "learning_rate": 6.301775147928994e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 625 + }, + { + "completion_length": 121.3203125, + "epoch": 0.37041420118343193, + "grad_norm": 0.4575230065753719, + "kl": 0.0498046875, + "learning_rate": 6.29585798816568e-07, + "loss": 0.002, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 626 + }, + { + "completion_length": 120.890625, + "epoch": 0.37100591715976333, + "grad_norm": 0.7461753255207615, + "kl": 0.036376953125, + "learning_rate": 6.289940828402367e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 627 + }, + { + "completion_length": 120.2265625, + "epoch": 0.3715976331360947, + "grad_norm": 0.51419092312948, + "kl": 0.045166015625, + "learning_rate": 6.284023668639052e-07, + "loss": 0.0018, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 628 + }, + { + "completion_length": 126.609375, + "epoch": 0.372189349112426, + "grad_norm": 0.5862509822993993, + "kl": 0.046875, + "learning_rate": 6.278106508875739e-07, + "loss": 0.0019, + "reward": 0.0546875, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 629 + }, + { + "completion_length": 129.609375, + "epoch": 0.3727810650887574, + "grad_norm": 0.3145312650124314, + "kl": 0.02685546875, + "learning_rate": 6.272189349112426e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 630 + }, + { + "completion_length": 120.9375, + "epoch": 0.37337278106508875, + "grad_norm": 0.423017615744333, + "kl": 0.0419921875, + "learning_rate": 6.266272189349113e-07, + "loss": 0.0017, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 631 + }, + { + "completion_length": 134.1875, + "epoch": 0.3739644970414201, + "grad_norm": 0.5013968270710079, + "kl": 0.0299072265625, + "learning_rate": 6.260355029585798e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 632 + }, + { + "completion_length": 117.828125, + "epoch": 0.3745562130177515, + "grad_norm": 0.5482650941299795, + "kl": 0.029541015625, + "learning_rate": 6.254437869822485e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 633 + }, + { + "completion_length": 122.84375, + "epoch": 0.37514792899408284, + "grad_norm": 0.7145234729032459, + "kl": 0.0302734375, + "learning_rate": 6.248520710059171e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 634 + }, + { + "completion_length": 124.34375, + "epoch": 0.3757396449704142, + "grad_norm": 1.0825098352759308, + "kl": 0.033935546875, + "learning_rate": 6.242603550295858e-07, + "loss": 0.0014, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 635 + }, + { + "completion_length": 127.3984375, + "epoch": 0.3763313609467456, + "grad_norm": 0.6058582619425739, + "kl": 0.02734375, + "learning_rate": 6.236686390532543e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 636 + }, + { + "completion_length": 123.015625, + "epoch": 0.3769230769230769, + "grad_norm": 0.6694035696081794, + "kl": 0.034423828125, + "learning_rate": 6.23076923076923e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 637 + }, + { + "completion_length": 120.5546875, + "epoch": 0.37751479289940826, + "grad_norm": 0.27623320783409044, + "kl": 0.0262451171875, + "learning_rate": 6.224852071005917e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 638 + }, + { + "completion_length": 125.4296875, + "epoch": 0.37810650887573966, + "grad_norm": 0.6801713604730955, + "kl": 0.033203125, + "learning_rate": 6.218934911242604e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 639 + }, + { + "completion_length": 118.265625, + "epoch": 0.378698224852071, + "grad_norm": 0.457805978118578, + "kl": 0.028076171875, + "learning_rate": 6.213017751479289e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 640 + }, + { + "completion_length": 127.5625, + "epoch": 0.3792899408284024, + "grad_norm": 0.364879204603429, + "kl": 0.025390625, + "learning_rate": 6.207100591715976e-07, + "loss": 0.001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 641 + }, + { + "completion_length": 122.6640625, + "epoch": 0.37988165680473374, + "grad_norm": 0.8850548618408604, + "kl": 0.037353515625, + "learning_rate": 6.201183431952662e-07, + "loss": 0.0015, + "reward": 0.1484375, + "reward_std": 0.11904378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 642 + }, + { + "completion_length": 129.546875, + "epoch": 0.3804733727810651, + "grad_norm": 0.4262759185382634, + "kl": 0.0301513671875, + "learning_rate": 6.195266272189349e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 643 + }, + { + "completion_length": 127.0234375, + "epoch": 0.3810650887573965, + "grad_norm": 0.4342387816342071, + "kl": 0.022216796875, + "learning_rate": 6.189349112426036e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 644 + }, + { + "completion_length": 119.2890625, + "epoch": 0.3816568047337278, + "grad_norm": 0.5777323128418748, + "kl": 0.0301513671875, + "learning_rate": 6.183431952662722e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 645 + }, + { + "completion_length": 122.0625, + "epoch": 0.38224852071005916, + "grad_norm": 0.5470324654216508, + "kl": 0.0286865234375, + "learning_rate": 6.177514792899408e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 646 + }, + { + "completion_length": 115.765625, + "epoch": 0.38284023668639056, + "grad_norm": 1.5719951312355998, + "kl": 0.03662109375, + "learning_rate": 6.171597633136094e-07, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 647 + }, + { + "completion_length": 125.515625, + "epoch": 0.3834319526627219, + "grad_norm": 0.5348712648896449, + "kl": 0.029052734375, + "learning_rate": 6.165680473372781e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 648 + }, + { + "completion_length": 116.65625, + "epoch": 0.38402366863905324, + "grad_norm": 0.8523621793715187, + "kl": 0.041015625, + "learning_rate": 6.159763313609467e-07, + "loss": 0.0016, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 649 + }, + { + "completion_length": 119.9296875, + "epoch": 0.38461538461538464, + "grad_norm": 0.6024743012184003, + "kl": 0.03662109375, + "learning_rate": 6.153846153846154e-07, + "loss": 0.0015, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 650 + }, + { + "completion_length": 114.8359375, + "epoch": 0.385207100591716, + "grad_norm": 0.7366125201911278, + "kl": 0.0341796875, + "learning_rate": 6.147928994082841e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 651 + }, + { + "completion_length": 116.7265625, + "epoch": 0.3857988165680473, + "grad_norm": 0.8708790412932383, + "kl": 0.0250244140625, + "learning_rate": 6.142011834319526e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 652 + }, + { + "completion_length": 122.8125, + "epoch": 0.3863905325443787, + "grad_norm": 0.5454306440343463, + "kl": 0.03662109375, + "learning_rate": 6.136094674556212e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 653 + }, + { + "completion_length": 122.828125, + "epoch": 0.38698224852071006, + "grad_norm": 0.740895477143288, + "kl": 0.0308837890625, + "learning_rate": 6.130177514792899e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.09858438372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 654 + }, + { + "completion_length": 118.3984375, + "epoch": 0.3875739644970414, + "grad_norm": 0.5510209793571246, + "kl": 0.03125, + "learning_rate": 6.124260355029586e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 655 + }, + { + "completion_length": 110.96875, + "epoch": 0.3881656804733728, + "grad_norm": 0.5547209911665342, + "kl": 0.0267333984375, + "learning_rate": 6.118343195266273e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 656 + }, + { + "completion_length": 117.4609375, + "epoch": 0.38875739644970414, + "grad_norm": 0.4367502362477481, + "kl": 0.0286865234375, + "learning_rate": 6.112426035502957e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 657 + }, + { + "completion_length": 126.3515625, + "epoch": 0.3893491124260355, + "grad_norm": 0.28099897616308867, + "kl": 0.038818359375, + "learning_rate": 6.106508875739644e-07, + "loss": 0.0016, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 658 + }, + { + "completion_length": 127.6484375, + "epoch": 0.3899408284023669, + "grad_norm": 0.5231241994211346, + "kl": 0.03564453125, + "learning_rate": 6.100591715976331e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 659 + }, + { + "completion_length": 116.625, + "epoch": 0.3905325443786982, + "grad_norm": 0.6239211836960648, + "kl": 0.0341796875, + "learning_rate": 6.094674556213018e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 660 + }, + { + "completion_length": 120.78125, + "epoch": 0.39112426035502956, + "grad_norm": 0.4313622231413516, + "kl": 0.029541015625, + "learning_rate": 6.088757396449704e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 661 + }, + { + "completion_length": 130.3359375, + "epoch": 0.39171597633136096, + "grad_norm": 0.34766662243920615, + "kl": 0.02490234375, + "learning_rate": 6.08284023668639e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 662 + }, + { + "completion_length": 137.125, + "epoch": 0.3923076923076923, + "grad_norm": 0.3970962211895298, + "kl": 0.022216796875, + "learning_rate": 6.076923076923076e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 663 + }, + { + "completion_length": 135.21875, + "epoch": 0.39289940828402364, + "grad_norm": 0.3244167444698933, + "kl": 0.0279541015625, + "learning_rate": 6.071005917159763e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 664 + }, + { + "completion_length": 122.2734375, + "epoch": 0.39349112426035504, + "grad_norm": 0.38713193294525844, + "kl": 0.028564453125, + "learning_rate": 6.065088757396449e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 665 + }, + { + "completion_length": 126.8828125, + "epoch": 0.3940828402366864, + "grad_norm": 0.6265318777307461, + "kl": 0.0281982421875, + "learning_rate": 6.059171597633136e-07, + "loss": 0.0011, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 666 + }, + { + "completion_length": 125.8125, + "epoch": 0.3946745562130177, + "grad_norm": 0.39713656887994964, + "kl": 0.0302734375, + "learning_rate": 6.053254437869822e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 667 + }, + { + "completion_length": 120.7265625, + "epoch": 0.3952662721893491, + "grad_norm": 6.6089380506554845, + "kl": 0.028076171875, + "learning_rate": 6.047337278106509e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 668 + }, + { + "completion_length": 125.5078125, + "epoch": 0.39585798816568046, + "grad_norm": 0.3999656840671447, + "kl": 0.0240478515625, + "learning_rate": 6.041420118343194e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 669 + }, + { + "completion_length": 123.7890625, + "epoch": 0.39644970414201186, + "grad_norm": 0.47707195431222604, + "kl": 0.02783203125, + "learning_rate": 6.035502958579881e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 670 + }, + { + "completion_length": 118.3671875, + "epoch": 0.3970414201183432, + "grad_norm": 0.5767154088156212, + "kl": 0.032958984375, + "learning_rate": 6.029585798816568e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 671 + }, + { + "completion_length": 120.578125, + "epoch": 0.39763313609467454, + "grad_norm": 0.5706707758518742, + "kl": 0.03466796875, + "learning_rate": 6.023668639053255e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 672 + }, + { + "completion_length": 131.5390625, + "epoch": 0.39822485207100594, + "grad_norm": 0.4437231227310345, + "kl": 0.023193359375, + "learning_rate": 6.017751479289941e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 673 + }, + { + "completion_length": 144.8984375, + "epoch": 0.3988165680473373, + "grad_norm": 0.5378406641800215, + "kl": 0.0247802734375, + "learning_rate": 6.011834319526627e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 674 + }, + { + "completion_length": 121.5234375, + "epoch": 0.3994082840236686, + "grad_norm": 0.4833449969080501, + "kl": 0.02734375, + "learning_rate": 6.005917159763313e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 675 + }, + { + "completion_length": 134.4296875, + "epoch": 0.4, + "grad_norm": 0.5254855379413707, + "kl": 0.02880859375, + "learning_rate": 6e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 676 + }, + { + "completion_length": 132.6015625, + "epoch": 0.40059171597633136, + "grad_norm": 0.5348293527066087, + "kl": 0.031982421875, + "learning_rate": 5.994082840236687e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 677 + }, + { + "completion_length": 117.421875, + "epoch": 0.4011834319526627, + "grad_norm": 0.7282861117470899, + "kl": 0.04541015625, + "learning_rate": 5.988165680473372e-07, + "loss": 0.0018, + "reward": 0.1640625, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 678 + }, + { + "completion_length": 115.5859375, + "epoch": 0.4017751479289941, + "grad_norm": 0.5357733015670105, + "kl": 0.0291748046875, + "learning_rate": 5.982248520710059e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 679 + }, + { + "completion_length": 125.8671875, + "epoch": 0.40236686390532544, + "grad_norm": 0.5607032306735985, + "kl": 0.0380859375, + "learning_rate": 5.976331360946746e-07, + "loss": 0.0015, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 680 + }, + { + "completion_length": 130.8203125, + "epoch": 0.4029585798816568, + "grad_norm": 0.6445133600021434, + "kl": 0.030029296875, + "learning_rate": 5.970414201183432e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 681 + }, + { + "completion_length": 142.171875, + "epoch": 0.4035502958579882, + "grad_norm": 0.5522045918849473, + "kl": 0.03759765625, + "learning_rate": 5.964497041420118e-07, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 682 + }, + { + "completion_length": 120.75, + "epoch": 0.4041420118343195, + "grad_norm": 0.5548335911477672, + "kl": 0.04296875, + "learning_rate": 5.958579881656804e-07, + "loss": 0.0017, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 683 + }, + { + "completion_length": 123.609375, + "epoch": 0.40473372781065087, + "grad_norm": 0.5106860736877541, + "kl": 0.0230712890625, + "learning_rate": 5.952662721893491e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 684 + }, + { + "completion_length": 135.234375, + "epoch": 0.40532544378698226, + "grad_norm": 0.6122467937490035, + "kl": 0.0341796875, + "learning_rate": 5.946745562130178e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 685 + }, + { + "completion_length": 132.3828125, + "epoch": 0.4059171597633136, + "grad_norm": 0.5039621209758229, + "kl": 0.029296875, + "learning_rate": 5.940828402366863e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 686 + }, + { + "completion_length": 126.734375, + "epoch": 0.40650887573964495, + "grad_norm": 1.1197649190071353, + "kl": 0.03759765625, + "learning_rate": 5.93491124260355e-07, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 687 + }, + { + "completion_length": 126.1953125, + "epoch": 0.40710059171597635, + "grad_norm": 0.8571151012012506, + "kl": 0.030517578125, + "learning_rate": 5.928994082840236e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 688 + }, + { + "completion_length": 132.3515625, + "epoch": 0.4076923076923077, + "grad_norm": 0.5390360231289163, + "kl": 0.028076171875, + "learning_rate": 5.923076923076923e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 689 + }, + { + "completion_length": 122.59375, + "epoch": 0.40828402366863903, + "grad_norm": 0.6025828269240733, + "kl": 0.0390625, + "learning_rate": 5.917159763313609e-07, + "loss": 0.0016, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 690 + }, + { + "completion_length": 124.0703125, + "epoch": 0.4088757396449704, + "grad_norm": 0.39300912863299176, + "kl": 0.0264892578125, + "learning_rate": 5.911242603550296e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 691 + }, + { + "completion_length": 125.2578125, + "epoch": 0.40946745562130177, + "grad_norm": 0.5562130954238642, + "kl": 0.0322265625, + "learning_rate": 5.905325443786982e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 692 + }, + { + "completion_length": 129.5078125, + "epoch": 0.4100591715976331, + "grad_norm": 0.4614709485141376, + "kl": 0.02734375, + "learning_rate": 5.899408284023668e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 693 + }, + { + "completion_length": 133.7265625, + "epoch": 0.4106508875739645, + "grad_norm": 0.3745919493327998, + "kl": 0.025146484375, + "learning_rate": 5.893491124260354e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 694 + }, + { + "completion_length": 129.4375, + "epoch": 0.41124260355029585, + "grad_norm": 0.3578236903038736, + "kl": 0.0191650390625, + "learning_rate": 5.887573964497041e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 695 + }, + { + "completion_length": 114.3671875, + "epoch": 0.41183431952662725, + "grad_norm": 0.3871354467250049, + "kl": 0.0216064453125, + "learning_rate": 5.881656804733728e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 696 + }, + { + "completion_length": 125.1171875, + "epoch": 0.4124260355029586, + "grad_norm": 0.7158344785040266, + "kl": 0.0301513671875, + "learning_rate": 5.875739644970415e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 697 + }, + { + "completion_length": 123.9140625, + "epoch": 0.41301775147928993, + "grad_norm": 0.4834866468581556, + "kl": 0.03369140625, + "learning_rate": 5.869822485207099e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 698 + }, + { + "completion_length": 129.4921875, + "epoch": 0.4136094674556213, + "grad_norm": 0.41196564547609726, + "kl": 0.025390625, + "learning_rate": 5.863905325443786e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 699 + }, + { + "completion_length": 127.03125, + "epoch": 0.41420118343195267, + "grad_norm": 0.4328720519882733, + "kl": 0.025146484375, + "learning_rate": 5.857988165680473e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 700 + }, + { + "completion_length": 129.796875, + "epoch": 0.414792899408284, + "grad_norm": 0.7031122226775093, + "kl": 0.026611328125, + "learning_rate": 5.85207100591716e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 701 + }, + { + "completion_length": 130.9765625, + "epoch": 0.4153846153846154, + "grad_norm": 0.47506311440849375, + "kl": 0.032470703125, + "learning_rate": 5.846153846153847e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 702 + }, + { + "completion_length": 124.125, + "epoch": 0.41597633136094675, + "grad_norm": 1.0300880288333871, + "kl": 0.038818359375, + "learning_rate": 5.840236686390532e-07, + "loss": 0.0016, + "reward": 0.2109375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2109375, + "step": 703 + }, + { + "completion_length": 132.546875, + "epoch": 0.4165680473372781, + "grad_norm": 0.30860200056148146, + "kl": 0.0235595703125, + "learning_rate": 5.834319526627218e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 704 + }, + { + "completion_length": 129.46875, + "epoch": 0.4171597633136095, + "grad_norm": 0.2617476945475117, + "kl": 0.0191650390625, + "learning_rate": 5.828402366863905e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 705 + }, + { + "completion_length": 114.484375, + "epoch": 0.41775147928994083, + "grad_norm": 0.6372597753481782, + "kl": 0.03857421875, + "learning_rate": 5.822485207100592e-07, + "loss": 0.0015, + "reward": 0.1484375, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 706 + }, + { + "completion_length": 128.2734375, + "epoch": 0.41834319526627217, + "grad_norm": 0.7255946235402204, + "kl": 0.0306396484375, + "learning_rate": 5.816568047337278e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 707 + }, + { + "completion_length": 130.453125, + "epoch": 0.41893491124260357, + "grad_norm": 0.4674137874604985, + "kl": 0.026123046875, + "learning_rate": 5.810650887573965e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 708 + }, + { + "completion_length": 129.09375, + "epoch": 0.4195266272189349, + "grad_norm": 0.5083012472922417, + "kl": 0.027099609375, + "learning_rate": 5.80473372781065e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 709 + }, + { + "completion_length": 120.921875, + "epoch": 0.42011834319526625, + "grad_norm": 0.4876016881737683, + "kl": 0.037109375, + "learning_rate": 5.798816568047337e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 710 + }, + { + "completion_length": 129.0234375, + "epoch": 0.42071005917159765, + "grad_norm": 0.5603566896810465, + "kl": 0.0220947265625, + "learning_rate": 5.792899408284023e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 711 + }, + { + "completion_length": 129.3125, + "epoch": 0.421301775147929, + "grad_norm": 0.5916714151358584, + "kl": 0.0245361328125, + "learning_rate": 5.78698224852071e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 712 + }, + { + "completion_length": 128.6640625, + "epoch": 0.42189349112426033, + "grad_norm": 0.8131419860650816, + "kl": 0.027587890625, + "learning_rate": 5.781065088757397e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 713 + }, + { + "completion_length": 122.734375, + "epoch": 0.42248520710059173, + "grad_norm": 0.06765884319149193, + "kl": 0.0296630859375, + "learning_rate": 5.775147928994083e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 714 + }, + { + "completion_length": 119.0859375, + "epoch": 0.4230769230769231, + "grad_norm": 0.658565138269454, + "kl": 0.0308837890625, + "learning_rate": 5.769230769230768e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 715 + }, + { + "completion_length": 124.7421875, + "epoch": 0.4236686390532544, + "grad_norm": 0.2815490678187719, + "kl": 0.025146484375, + "learning_rate": 5.763313609467455e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 716 + }, + { + "completion_length": 131.6640625, + "epoch": 0.4242603550295858, + "grad_norm": 0.5620129015832279, + "kl": 0.024658203125, + "learning_rate": 5.757396449704142e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 717 + }, + { + "completion_length": 122.0390625, + "epoch": 0.42485207100591715, + "grad_norm": 0.49857398452816654, + "kl": 0.03173828125, + "learning_rate": 5.751479289940829e-07, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 718 + }, + { + "completion_length": 118.6875, + "epoch": 0.4254437869822485, + "grad_norm": 0.4379481646497665, + "kl": 0.0267333984375, + "learning_rate": 5.745562130177514e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 719 + }, + { + "completion_length": 122.875, + "epoch": 0.4260355029585799, + "grad_norm": 0.7662416241891038, + "kl": 0.032958984375, + "learning_rate": 5.739644970414201e-07, + "loss": 0.0013, + "reward": 0.203125, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.203125, + "step": 720 + }, + { + "completion_length": 129.9453125, + "epoch": 0.42662721893491123, + "grad_norm": 0.17179288832850495, + "kl": 0.026611328125, + "learning_rate": 5.733727810650887e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 721 + }, + { + "completion_length": 115.5703125, + "epoch": 0.4272189349112426, + "grad_norm": 0.6232200682967399, + "kl": 0.03662109375, + "learning_rate": 5.727810650887574e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 722 + }, + { + "completion_length": 130.5703125, + "epoch": 0.427810650887574, + "grad_norm": 3.4121664961247413, + "kl": 0.02294921875, + "learning_rate": 5.72189349112426e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 723 + }, + { + "completion_length": 119.0234375, + "epoch": 0.4284023668639053, + "grad_norm": 0.6013837734931465, + "kl": 0.0267333984375, + "learning_rate": 5.715976331360946e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 724 + }, + { + "completion_length": 120.125, + "epoch": 0.4289940828402367, + "grad_norm": 0.7105313077649819, + "kl": 0.036376953125, + "learning_rate": 5.710059171597633e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 725 + }, + { + "completion_length": 126.5859375, + "epoch": 0.42958579881656805, + "grad_norm": 0.5769191207504994, + "kl": 0.04443359375, + "learning_rate": 5.70414201183432e-07, + "loss": 0.0018, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 726 + }, + { + "completion_length": 124.640625, + "epoch": 0.4301775147928994, + "grad_norm": 0.5319485039850909, + "kl": 0.0250244140625, + "learning_rate": 5.698224852071005e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 727 + }, + { + "completion_length": 122.6015625, + "epoch": 0.4307692307692308, + "grad_norm": 0.43191426947583755, + "kl": 0.024169921875, + "learning_rate": 5.692307692307692e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 728 + }, + { + "completion_length": 115.375, + "epoch": 0.43136094674556213, + "grad_norm": 0.24055350487442487, + "kl": 0.03125, + "learning_rate": 5.686390532544378e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 729 + }, + { + "completion_length": 118.8984375, + "epoch": 0.4319526627218935, + "grad_norm": 0.8283199940585683, + "kl": 0.0277099609375, + "learning_rate": 5.680473372781065e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 730 + }, + { + "completion_length": 122.5078125, + "epoch": 0.4325443786982249, + "grad_norm": 1.2796290075457826, + "kl": 0.0380859375, + "learning_rate": 5.674556213017752e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 731 + }, + { + "completion_length": 123.0, + "epoch": 0.4331360946745562, + "grad_norm": 0.2835717799372122, + "kl": 0.0184326171875, + "learning_rate": 5.668639053254438e-07, + "loss": 0.0007, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 732 + }, + { + "completion_length": 111.3203125, + "epoch": 0.43372781065088756, + "grad_norm": 0.6919746169052234, + "kl": 0.043701171875, + "learning_rate": 5.662721893491124e-07, + "loss": 0.0017, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 733 + }, + { + "completion_length": 114.953125, + "epoch": 0.43431952662721895, + "grad_norm": 1.0772505669541428, + "kl": 0.03369140625, + "learning_rate": 5.65680473372781e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.11420939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 734 + }, + { + "completion_length": 116.9296875, + "epoch": 0.4349112426035503, + "grad_norm": 0.40788572741115753, + "kl": 0.034912109375, + "learning_rate": 5.650887573964497e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 735 + }, + { + "completion_length": 130.296875, + "epoch": 0.43550295857988164, + "grad_norm": 0.056527515166868776, + "kl": 0.02099609375, + "learning_rate": 5.644970414201183e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 736 + }, + { + "completion_length": 116.859375, + "epoch": 0.43609467455621304, + "grad_norm": 0.6952003792004148, + "kl": 0.03515625, + "learning_rate": 5.63905325443787e-07, + "loss": 0.0014, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 737 + }, + { + "completion_length": 119.8515625, + "epoch": 0.4366863905325444, + "grad_norm": 0.3134927312556443, + "kl": 0.026123046875, + "learning_rate": 5.633136094674556e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 738 + }, + { + "completion_length": 121.390625, + "epoch": 0.4372781065088757, + "grad_norm": 0.5847146389041795, + "kl": 0.0284423828125, + "learning_rate": 5.627218934911242e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 739 + }, + { + "completion_length": 113.1484375, + "epoch": 0.4378698224852071, + "grad_norm": 1.4213713858675996, + "kl": 0.033203125, + "learning_rate": 5.621301775147928e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.11662658303976059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 740 + }, + { + "completion_length": 118.796875, + "epoch": 0.43846153846153846, + "grad_norm": 0.06415571667570931, + "kl": 0.03271484375, + "learning_rate": 5.615384615384615e-07, + "loss": 0.0013, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 741 + }, + { + "completion_length": 121.265625, + "epoch": 0.4390532544378698, + "grad_norm": 0.8410198237209259, + "kl": 0.03125, + "learning_rate": 5.609467455621302e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 742 + }, + { + "completion_length": 123.2109375, + "epoch": 0.4396449704142012, + "grad_norm": 0.7264362066405277, + "kl": 0.035888671875, + "learning_rate": 5.603550295857989e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 743 + }, + { + "completion_length": 120.765625, + "epoch": 0.44023668639053254, + "grad_norm": 0.3259116739732085, + "kl": 0.0234375, + "learning_rate": 5.597633136094673e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 744 + }, + { + "completion_length": 124.546875, + "epoch": 0.4408284023668639, + "grad_norm": 0.3578737802682367, + "kl": 0.0322265625, + "learning_rate": 5.59171597633136e-07, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 745 + }, + { + "completion_length": 123.03125, + "epoch": 0.4414201183431953, + "grad_norm": 0.4125460192018465, + "kl": 0.0277099609375, + "learning_rate": 5.585798816568047e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 746 + }, + { + "completion_length": 117.3046875, + "epoch": 0.4420118343195266, + "grad_norm": 0.7535330279870045, + "kl": 0.035888671875, + "learning_rate": 5.579881656804734e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 747 + }, + { + "completion_length": 117.1015625, + "epoch": 0.44260355029585796, + "grad_norm": 1.9810957835719456, + "kl": 0.034912109375, + "learning_rate": 5.57396449704142e-07, + "loss": 0.0014, + "reward": 0.0625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 748 + }, + { + "completion_length": 116.3671875, + "epoch": 0.44319526627218936, + "grad_norm": 0.7170806769714086, + "kl": 0.02734375, + "learning_rate": 5.568047337278107e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 749 + }, + { + "completion_length": 120.7734375, + "epoch": 0.4437869822485207, + "grad_norm": 0.6321891585743961, + "kl": 0.026123046875, + "learning_rate": 5.562130177514792e-07, + "loss": 0.001, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 750 + }, + { + "completion_length": 118.4296875, + "epoch": 0.4443786982248521, + "grad_norm": 0.6208073887995769, + "kl": 0.036865234375, + "learning_rate": 5.556213017751479e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 751 + }, + { + "completion_length": 117.171875, + "epoch": 0.44497041420118344, + "grad_norm": 0.5742075540089914, + "kl": 0.035888671875, + "learning_rate": 5.550295857988165e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 752 + }, + { + "completion_length": 122.4296875, + "epoch": 0.4455621301775148, + "grad_norm": 0.2952524695085959, + "kl": 0.03564453125, + "learning_rate": 5.544378698224852e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 753 + }, + { + "completion_length": 109.5390625, + "epoch": 0.4461538461538462, + "grad_norm": 0.6457601431604845, + "kl": 0.040771484375, + "learning_rate": 5.538461538461539e-07, + "loss": 0.0016, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 754 + }, + { + "completion_length": 116.734375, + "epoch": 0.4467455621301775, + "grad_norm": 1.908623523460671, + "kl": 0.0400390625, + "learning_rate": 5.532544378698225e-07, + "loss": 0.0016, + "reward": 0.1015625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 755 + }, + { + "completion_length": 110.578125, + "epoch": 0.44733727810650886, + "grad_norm": 0.5767143350712665, + "kl": 0.0400390625, + "learning_rate": 5.52662721893491e-07, + "loss": 0.0016, + "reward": 0.1953125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 756 + }, + { + "completion_length": 110.4921875, + "epoch": 0.44792899408284026, + "grad_norm": 0.3278284097054489, + "kl": 0.03759765625, + "learning_rate": 5.520710059171597e-07, + "loss": 0.0015, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 757 + }, + { + "completion_length": 111.546875, + "epoch": 0.4485207100591716, + "grad_norm": 0.5758526673102218, + "kl": 0.03369140625, + "learning_rate": 5.514792899408284e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 758 + }, + { + "completion_length": 120.75, + "epoch": 0.44911242603550294, + "grad_norm": 0.7868011152280301, + "kl": 0.024169921875, + "learning_rate": 5.508875739644971e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 759 + }, + { + "completion_length": 121.7890625, + "epoch": 0.44970414201183434, + "grad_norm": 0.3222265598671648, + "kl": 0.026123046875, + "learning_rate": 5.502958579881657e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 760 + }, + { + "completion_length": 114.578125, + "epoch": 0.4502958579881657, + "grad_norm": 1.1169040652243958, + "kl": 0.0291748046875, + "learning_rate": 5.497041420118343e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 761 + }, + { + "completion_length": 113.6171875, + "epoch": 0.450887573964497, + "grad_norm": 0.740491729299172, + "kl": 0.0269775390625, + "learning_rate": 5.491124260355029e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 762 + }, + { + "completion_length": 116.3515625, + "epoch": 0.4514792899408284, + "grad_norm": 0.7052487572479517, + "kl": 0.03662109375, + "learning_rate": 5.485207100591716e-07, + "loss": 0.0015, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 763 + }, + { + "completion_length": 117.1328125, + "epoch": 0.45207100591715976, + "grad_norm": 0.5902588201008392, + "kl": 0.0322265625, + "learning_rate": 5.479289940828403e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 764 + }, + { + "completion_length": 122.3984375, + "epoch": 0.4526627218934911, + "grad_norm": 0.3101819287398495, + "kl": 0.0306396484375, + "learning_rate": 5.473372781065088e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 765 + }, + { + "completion_length": 120.5546875, + "epoch": 0.4532544378698225, + "grad_norm": 0.6358825567116966, + "kl": 0.0272216796875, + "learning_rate": 5.467455621301775e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 766 + }, + { + "completion_length": 129.4765625, + "epoch": 0.45384615384615384, + "grad_norm": 0.640064818672871, + "kl": 0.031005859375, + "learning_rate": 5.461538461538461e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 767 + }, + { + "completion_length": 119.6015625, + "epoch": 0.4544378698224852, + "grad_norm": 0.4179507497344744, + "kl": 0.032470703125, + "learning_rate": 5.455621301775148e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 768 + }, + { + "completion_length": 116.2578125, + "epoch": 0.4550295857988166, + "grad_norm": 0.2856973728571418, + "kl": 0.022216796875, + "learning_rate": 5.449704142011834e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 769 + }, + { + "completion_length": 128.671875, + "epoch": 0.4556213017751479, + "grad_norm": 0.3846538571271079, + "kl": 0.0289306640625, + "learning_rate": 5.44378698224852e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 770 + }, + { + "completion_length": 120.265625, + "epoch": 0.45621301775147927, + "grad_norm": 0.40311879485273455, + "kl": 0.0380859375, + "learning_rate": 5.437869822485207e-07, + "loss": 0.0015, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 771 + }, + { + "completion_length": 126.3984375, + "epoch": 0.45680473372781066, + "grad_norm": 0.22815244804273238, + "kl": 0.0252685546875, + "learning_rate": 5.431952662721894e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 772 + }, + { + "completion_length": 131.53125, + "epoch": 0.457396449704142, + "grad_norm": 0.4338812231852395, + "kl": 0.029052734375, + "learning_rate": 5.426035502958579e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 773 + }, + { + "completion_length": 128.0234375, + "epoch": 0.45798816568047335, + "grad_norm": 0.08168918202050264, + "kl": 0.02490234375, + "learning_rate": 5.420118343195266e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 774 + }, + { + "completion_length": 125.1875, + "epoch": 0.45857988165680474, + "grad_norm": 0.5289696590756887, + "kl": 0.023193359375, + "learning_rate": 5.414201183431952e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 775 + }, + { + "completion_length": 116.1796875, + "epoch": 0.4591715976331361, + "grad_norm": 0.35609824056805994, + "kl": 0.0322265625, + "learning_rate": 5.408284023668639e-07, + "loss": 0.0013, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 776 + }, + { + "completion_length": 108.421875, + "epoch": 0.4597633136094675, + "grad_norm": 0.46525080592242696, + "kl": 0.029296875, + "learning_rate": 5.402366863905325e-07, + "loss": 0.0012, + "reward": 0.171875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 777 + }, + { + "completion_length": 120.2734375, + "epoch": 0.4603550295857988, + "grad_norm": 0.29238810689849803, + "kl": 0.03466796875, + "learning_rate": 5.396449704142012e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 778 + }, + { + "completion_length": 121.609375, + "epoch": 0.46094674556213017, + "grad_norm": 0.6124912103811252, + "kl": 0.0272216796875, + "learning_rate": 5.390532544378698e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 779 + }, + { + "completion_length": 120.40625, + "epoch": 0.46153846153846156, + "grad_norm": 0.4020182265083479, + "kl": 0.029296875, + "learning_rate": 5.384615384615384e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.078125, + "step": 780 + }, + { + "completion_length": 127.421875, + "epoch": 0.4621301775147929, + "grad_norm": 0.5795044191323172, + "kl": 0.026611328125, + "learning_rate": 5.37869822485207e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 781 + }, + { + "completion_length": 111.7734375, + "epoch": 0.46272189349112425, + "grad_norm": 0.3108931753279031, + "kl": 0.02978515625, + "learning_rate": 5.372781065088757e-07, + "loss": 0.0012, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 782 + }, + { + "completion_length": 121.5078125, + "epoch": 0.46331360946745564, + "grad_norm": 0.761548822975496, + "kl": 0.0380859375, + "learning_rate": 5.366863905325444e-07, + "loss": 0.0015, + "reward": 0.1640625, + "reward_std": 0.11420938372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 783 + }, + { + "completion_length": 122.4140625, + "epoch": 0.463905325443787, + "grad_norm": 0.535652821056829, + "kl": 0.030517578125, + "learning_rate": 5.360946745562131e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 784 + }, + { + "completion_length": 132.1640625, + "epoch": 0.46449704142011833, + "grad_norm": 1.353730070548512, + "kl": 0.028564453125, + "learning_rate": 5.355029585798815e-07, + "loss": 0.0011, + "reward": 0.171875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 785 + }, + { + "completion_length": 129.796875, + "epoch": 0.4650887573964497, + "grad_norm": 0.3937349796345773, + "kl": 0.0244140625, + "learning_rate": 5.349112426035502e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 786 + }, + { + "completion_length": 133.8984375, + "epoch": 0.46568047337278107, + "grad_norm": 0.2908823179771442, + "kl": 0.02587890625, + "learning_rate": 5.343195266272189e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 787 + }, + { + "completion_length": 126.4921875, + "epoch": 0.4662721893491124, + "grad_norm": 0.7053663024060098, + "kl": 0.0311279296875, + "learning_rate": 5.337278106508876e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 788 + }, + { + "completion_length": 122.625, + "epoch": 0.4668639053254438, + "grad_norm": 3.6752407052276266, + "kl": 0.03466796875, + "learning_rate": 5.331360946745563e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 789 + }, + { + "completion_length": 127.140625, + "epoch": 0.46745562130177515, + "grad_norm": 0.2468593297865429, + "kl": 0.0286865234375, + "learning_rate": 5.325443786982249e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 790 + }, + { + "completion_length": 133.375, + "epoch": 0.4680473372781065, + "grad_norm": 0.38954007556961473, + "kl": 0.025146484375, + "learning_rate": 5.319526627218934e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 791 + }, + { + "completion_length": 129.40625, + "epoch": 0.4686390532544379, + "grad_norm": 0.08843091331741072, + "kl": 0.032958984375, + "learning_rate": 5.313609467455621e-07, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 792 + }, + { + "completion_length": 129.828125, + "epoch": 0.46923076923076923, + "grad_norm": 0.058294855307931596, + "kl": 0.0245361328125, + "learning_rate": 5.307692307692308e-07, + "loss": 0.001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 793 + }, + { + "completion_length": 131.1171875, + "epoch": 0.46982248520710057, + "grad_norm": 0.6892860195241568, + "kl": 0.03076171875, + "learning_rate": 5.301775147928994e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 794 + }, + { + "completion_length": 124.7421875, + "epoch": 0.47041420118343197, + "grad_norm": 0.4997522235636295, + "kl": 0.02392578125, + "learning_rate": 5.295857988165681e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 795 + }, + { + "completion_length": 132.8046875, + "epoch": 0.4710059171597633, + "grad_norm": 0.5554042426785889, + "kl": 0.0303955078125, + "learning_rate": 5.289940828402366e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 796 + }, + { + "completion_length": 123.5703125, + "epoch": 0.47159763313609465, + "grad_norm": 0.6218280224215168, + "kl": 0.03076171875, + "learning_rate": 5.284023668639053e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 797 + }, + { + "completion_length": 128.1796875, + "epoch": 0.47218934911242605, + "grad_norm": 0.5937968466267416, + "kl": 0.023681640625, + "learning_rate": 5.278106508875739e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 798 + }, + { + "completion_length": 138.734375, + "epoch": 0.4727810650887574, + "grad_norm": 0.4605697233472056, + "kl": 0.017822265625, + "learning_rate": 5.272189349112426e-07, + "loss": 0.0007, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 799 + }, + { + "completion_length": 126.78125, + "epoch": 0.47337278106508873, + "grad_norm": 0.775186332952182, + "kl": 0.03466796875, + "learning_rate": 5.266272189349113e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 800 + }, + { + "completion_length": 134.65625, + "epoch": 0.47396449704142013, + "grad_norm": 0.8240471027001092, + "kl": 0.022216796875, + "learning_rate": 5.260355029585799e-07, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 801 + }, + { + "completion_length": 131.4375, + "epoch": 0.47455621301775147, + "grad_norm": 0.6805462707969241, + "kl": 0.035400390625, + "learning_rate": 5.254437869822484e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 802 + }, + { + "completion_length": 129.8359375, + "epoch": 0.4751479289940828, + "grad_norm": 0.6299917897995363, + "kl": 0.03662109375, + "learning_rate": 5.248520710059171e-07, + "loss": 0.0015, + "reward": 0.203125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.203125, + "step": 803 + }, + { + "completion_length": 128.328125, + "epoch": 0.4757396449704142, + "grad_norm": 0.49593919299273853, + "kl": 0.0361328125, + "learning_rate": 5.242603550295858e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 804 + }, + { + "completion_length": 124.5546875, + "epoch": 0.47633136094674555, + "grad_norm": 0.31436728668579816, + "kl": 0.0238037109375, + "learning_rate": 5.236686390532545e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 805 + }, + { + "completion_length": 122.390625, + "epoch": 0.47692307692307695, + "grad_norm": 0.286875005673474, + "kl": 0.030029296875, + "learning_rate": 5.23076923076923e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 806 + }, + { + "completion_length": 139.3125, + "epoch": 0.4775147928994083, + "grad_norm": 0.677500172151261, + "kl": 0.0255126953125, + "learning_rate": 5.224852071005917e-07, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 807 + }, + { + "completion_length": 123.578125, + "epoch": 0.47810650887573963, + "grad_norm": 0.6350467890283358, + "kl": 0.03369140625, + "learning_rate": 5.218934911242603e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 808 + }, + { + "completion_length": 120.4453125, + "epoch": 0.47869822485207103, + "grad_norm": 1.112274912601818, + "kl": 0.0311279296875, + "learning_rate": 5.21301775147929e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 809 + }, + { + "completion_length": 123.5546875, + "epoch": 0.47928994082840237, + "grad_norm": 0.5705705277679465, + "kl": 0.031494140625, + "learning_rate": 5.207100591715976e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.1171875, + "step": 810 + }, + { + "completion_length": 128.1015625, + "epoch": 0.4798816568047337, + "grad_norm": 0.8779669165833008, + "kl": 0.035400390625, + "learning_rate": 5.201183431952662e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.13466878235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 811 + }, + { + "completion_length": 121.5625, + "epoch": 0.4804733727810651, + "grad_norm": 0.45383409019945503, + "kl": 0.034912109375, + "learning_rate": 5.195266272189349e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 812 + }, + { + "completion_length": 130.125, + "epoch": 0.48106508875739645, + "grad_norm": 0.5641098809443786, + "kl": 0.031982421875, + "learning_rate": 5.189349112426036e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 813 + }, + { + "completion_length": 128.265625, + "epoch": 0.4816568047337278, + "grad_norm": 0.10314334079342331, + "kl": 0.023193359375, + "learning_rate": 5.183431952662721e-07, + "loss": 0.0009, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 814 + }, + { + "completion_length": 122.7265625, + "epoch": 0.4822485207100592, + "grad_norm": 0.47666358930827113, + "kl": 0.033203125, + "learning_rate": 5.177514792899408e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 815 + }, + { + "completion_length": 126.5703125, + "epoch": 0.48284023668639053, + "grad_norm": 0.6020069772708556, + "kl": 0.034912109375, + "learning_rate": 5.171597633136094e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 816 + }, + { + "completion_length": 128.109375, + "epoch": 0.4834319526627219, + "grad_norm": 0.49485204699289603, + "kl": 0.026123046875, + "learning_rate": 5.165680473372781e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 817 + }, + { + "completion_length": 127.640625, + "epoch": 0.4840236686390533, + "grad_norm": 0.524679667560044, + "kl": 0.0242919921875, + "learning_rate": 5.159763313609468e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 818 + }, + { + "completion_length": 124.2890625, + "epoch": 0.4846153846153846, + "grad_norm": 0.061947643998446114, + "kl": 0.018798828125, + "learning_rate": 5.153846153846153e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 819 + }, + { + "completion_length": 121.5625, + "epoch": 0.48520710059171596, + "grad_norm": 4.095994776226779, + "kl": 0.042236328125, + "learning_rate": 5.14792899408284e-07, + "loss": 0.0017, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 820 + }, + { + "completion_length": 129.8359375, + "epoch": 0.48579881656804735, + "grad_norm": 0.8180757339461693, + "kl": 0.03076171875, + "learning_rate": 5.142011834319526e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 821 + }, + { + "completion_length": 120.453125, + "epoch": 0.4863905325443787, + "grad_norm": 0.061086633265337635, + "kl": 0.03271484375, + "learning_rate": 5.136094674556213e-07, + "loss": 0.0013, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 822 + }, + { + "completion_length": 138.046875, + "epoch": 0.48698224852071004, + "grad_norm": 0.5774663036663364, + "kl": 0.02587890625, + "learning_rate": 5.130177514792899e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 823 + }, + { + "completion_length": 120.28125, + "epoch": 0.48757396449704143, + "grad_norm": 0.7728813620485478, + "kl": 0.041015625, + "learning_rate": 5.124260355029586e-07, + "loss": 0.0016, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 824 + }, + { + "completion_length": 116.625, + "epoch": 0.4881656804733728, + "grad_norm": 0.5985038966547123, + "kl": 0.0400390625, + "learning_rate": 5.118343195266272e-07, + "loss": 0.0016, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 825 + }, + { + "completion_length": 117.4921875, + "epoch": 0.4887573964497041, + "grad_norm": 0.43005375052139405, + "kl": 0.033203125, + "learning_rate": 5.112426035502958e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 826 + }, + { + "completion_length": 116.140625, + "epoch": 0.4893491124260355, + "grad_norm": 0.5330269284009703, + "kl": 0.033935546875, + "learning_rate": 5.106508875739644e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 827 + }, + { + "completion_length": 125.328125, + "epoch": 0.48994082840236686, + "grad_norm": 0.23650136767694874, + "kl": 0.029296875, + "learning_rate": 5.100591715976331e-07, + "loss": 0.0012, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 828 + }, + { + "completion_length": 123.40625, + "epoch": 0.4905325443786982, + "grad_norm": 0.5721485117693927, + "kl": 0.032470703125, + "learning_rate": 5.094674556213018e-07, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 829 + }, + { + "completion_length": 112.53125, + "epoch": 0.4911242603550296, + "grad_norm": 0.4865418100335117, + "kl": 0.032470703125, + "learning_rate": 5.088757396449705e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 830 + }, + { + "completion_length": 127.9453125, + "epoch": 0.49171597633136094, + "grad_norm": 0.05919609533758626, + "kl": 0.019775390625, + "learning_rate": 5.082840236686389e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 831 + }, + { + "completion_length": 120.703125, + "epoch": 0.49230769230769234, + "grad_norm": 0.3429902668768904, + "kl": 0.0247802734375, + "learning_rate": 5.076923076923076e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 832 + }, + { + "completion_length": 119.5546875, + "epoch": 0.4928994082840237, + "grad_norm": 0.6105745135188593, + "kl": 0.033447265625, + "learning_rate": 5.071005917159763e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 833 + }, + { + "completion_length": 123.859375, + "epoch": 0.493491124260355, + "grad_norm": 0.557935755618462, + "kl": 0.032470703125, + "learning_rate": 5.06508875739645e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 834 + }, + { + "completion_length": 125.78125, + "epoch": 0.4940828402366864, + "grad_norm": 0.675535457268183, + "kl": 0.036865234375, + "learning_rate": 5.059171597633136e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 835 + }, + { + "completion_length": 119.7578125, + "epoch": 0.49467455621301776, + "grad_norm": 0.5431134777380299, + "kl": 0.02978515625, + "learning_rate": 5.053254437869823e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 836 + }, + { + "completion_length": 119.78125, + "epoch": 0.4952662721893491, + "grad_norm": 1.221177656508275, + "kl": 0.04052734375, + "learning_rate": 5.047337278106508e-07, + "loss": 0.0016, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 837 + }, + { + "completion_length": 121.015625, + "epoch": 0.4958579881656805, + "grad_norm": 0.4816770325652796, + "kl": 0.03857421875, + "learning_rate": 5.041420118343195e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 838 + }, + { + "completion_length": 122.8125, + "epoch": 0.49644970414201184, + "grad_norm": 0.6258502997624382, + "kl": 0.036865234375, + "learning_rate": 5.035502958579881e-07, + "loss": 0.0015, + "reward": 0.1640625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 839 + }, + { + "completion_length": 126.0625, + "epoch": 0.4970414201183432, + "grad_norm": 0.5475227468784842, + "kl": 0.0322265625, + "learning_rate": 5.029585798816568e-07, + "loss": 0.0013, + "reward": 0.15625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 840 + }, + { + "completion_length": 133.3125, + "epoch": 0.4976331360946746, + "grad_norm": 0.35740281271898733, + "kl": 0.024169921875, + "learning_rate": 5.023668639053255e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 841 + }, + { + "completion_length": 135.140625, + "epoch": 0.4982248520710059, + "grad_norm": 0.11666111522804415, + "kl": 0.0224609375, + "learning_rate": 5.017751479289941e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 842 + }, + { + "completion_length": 134.9921875, + "epoch": 0.49881656804733726, + "grad_norm": 0.30448046744400675, + "kl": 0.0244140625, + "learning_rate": 5.011834319526626e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 843 + }, + { + "completion_length": 125.296875, + "epoch": 0.49940828402366866, + "grad_norm": 0.4466705186951785, + "kl": 0.0283203125, + "learning_rate": 5.005917159763313e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 844 + }, + { + "completion_length": 119.25, + "epoch": 0.5, + "grad_norm": 0.5097646748231638, + "kl": 0.0252685546875, + "learning_rate": 5e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 845 + }, + { + "completion_length": 124.2265625, + "epoch": 0.5005917159763313, + "grad_norm": 0.4647631938886382, + "kl": 0.03759765625, + "learning_rate": 4.994082840236687e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 846 + }, + { + "completion_length": 121.25, + "epoch": 0.5011834319526627, + "grad_norm": 0.5489259090413676, + "kl": 0.0308837890625, + "learning_rate": 4.988165680473372e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 847 + }, + { + "completion_length": 113.7265625, + "epoch": 0.5017751479289941, + "grad_norm": 0.9003706017376398, + "kl": 0.046875, + "learning_rate": 4.982248520710059e-07, + "loss": 0.0019, + "reward": 0.1875, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 848 + }, + { + "completion_length": 128.859375, + "epoch": 0.5023668639053255, + "grad_norm": 0.4032786103663206, + "kl": 0.031494140625, + "learning_rate": 4.976331360946746e-07, + "loss": 0.0013, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 849 + }, + { + "completion_length": 120.296875, + "epoch": 0.5029585798816568, + "grad_norm": 0.6353809475780103, + "kl": 0.025146484375, + "learning_rate": 4.970414201183432e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 850 + }, + { + "completion_length": 126.7109375, + "epoch": 0.5035502958579882, + "grad_norm": 0.36721209471762184, + "kl": 0.02685546875, + "learning_rate": 4.964497041420119e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 851 + }, + { + "completion_length": 119.2265625, + "epoch": 0.5041420118343195, + "grad_norm": 0.38390014478962836, + "kl": 0.027587890625, + "learning_rate": 4.958579881656804e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 852 + }, + { + "completion_length": 122.625, + "epoch": 0.5047337278106508, + "grad_norm": 0.7450672286337058, + "kl": 0.0308837890625, + "learning_rate": 4.952662721893491e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 853 + }, + { + "completion_length": 134.1953125, + "epoch": 0.5053254437869823, + "grad_norm": 0.3096466219796354, + "kl": 0.031005859375, + "learning_rate": 4.946745562130177e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 854 + }, + { + "completion_length": 125.234375, + "epoch": 0.5059171597633136, + "grad_norm": 0.10289629309266178, + "kl": 0.0233154296875, + "learning_rate": 4.940828402366864e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 855 + }, + { + "completion_length": 128.4453125, + "epoch": 0.506508875739645, + "grad_norm": 0.22158260420611542, + "kl": 0.0181884765625, + "learning_rate": 4.93491124260355e-07, + "loss": 0.0007, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 856 + }, + { + "completion_length": 139.1171875, + "epoch": 0.5071005917159763, + "grad_norm": 0.8138788862992246, + "kl": 0.0206298828125, + "learning_rate": 4.928994082840236e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 857 + }, + { + "completion_length": 124.7109375, + "epoch": 0.5076923076923077, + "grad_norm": 1.1970460300905932, + "kl": 0.03076171875, + "learning_rate": 4.923076923076923e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 858 + }, + { + "completion_length": 131.9296875, + "epoch": 0.508284023668639, + "grad_norm": 0.5733115814482879, + "kl": 0.0263671875, + "learning_rate": 4.917159763313609e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 859 + }, + { + "completion_length": 127.8359375, + "epoch": 0.5088757396449705, + "grad_norm": 0.6323441876194337, + "kl": 0.02587890625, + "learning_rate": 4.911242603550296e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 860 + }, + { + "completion_length": 127.2890625, + "epoch": 0.5094674556213018, + "grad_norm": 0.6340227819025883, + "kl": 0.0458984375, + "learning_rate": 4.905325443786982e-07, + "loss": 0.0018, + "reward": 0.1953125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 861 + }, + { + "completion_length": 128.6484375, + "epoch": 0.5100591715976331, + "grad_norm": 0.7724676816879534, + "kl": 0.03271484375, + "learning_rate": 4.899408284023668e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 862 + }, + { + "completion_length": 120.6171875, + "epoch": 0.5106508875739645, + "grad_norm": 0.3048525121410532, + "kl": 0.0283203125, + "learning_rate": 4.893491124260355e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 863 + }, + { + "completion_length": 125.796875, + "epoch": 0.5112426035502958, + "grad_norm": 1.0657171787685458, + "kl": 0.0267333984375, + "learning_rate": 4.887573964497041e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 864 + }, + { + "completion_length": 118.8359375, + "epoch": 0.5118343195266272, + "grad_norm": 0.512187081977303, + "kl": 0.036865234375, + "learning_rate": 4.881656804733728e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 865 + }, + { + "completion_length": 130.75, + "epoch": 0.5124260355029586, + "grad_norm": 0.4974643619096443, + "kl": 0.027587890625, + "learning_rate": 4.875739644970414e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 866 + }, + { + "completion_length": 124.6171875, + "epoch": 0.51301775147929, + "grad_norm": 0.4365361461997679, + "kl": 0.021240234375, + "learning_rate": 4.8698224852071e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 867 + }, + { + "completion_length": 130.1640625, + "epoch": 0.5136094674556213, + "grad_norm": 0.6049659179874515, + "kl": 0.03466796875, + "learning_rate": 4.863905325443786e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 868 + }, + { + "completion_length": 132.6875, + "epoch": 0.5142011834319526, + "grad_norm": 0.4167496862123745, + "kl": 0.020751953125, + "learning_rate": 4.857988165680473e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 869 + }, + { + "completion_length": 128.765625, + "epoch": 0.514792899408284, + "grad_norm": 0.7196610216355583, + "kl": 0.0279541015625, + "learning_rate": 4.852071005917159e-07, + "loss": 0.0011, + "reward": 0.1796875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 870 + }, + { + "completion_length": 136.328125, + "epoch": 0.5153846153846153, + "grad_norm": 0.3855338363064419, + "kl": 0.033203125, + "learning_rate": 4.846153846153846e-07, + "loss": 0.0013, + "reward": 0.15625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 871 + }, + { + "completion_length": 136.9453125, + "epoch": 0.5159763313609468, + "grad_norm": 0.5448005601884904, + "kl": 0.032958984375, + "learning_rate": 4.840236686390533e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 872 + }, + { + "completion_length": 121.4453125, + "epoch": 0.5165680473372781, + "grad_norm": 0.7666645630588869, + "kl": 0.034912109375, + "learning_rate": 4.834319526627218e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 873 + }, + { + "completion_length": 122.6640625, + "epoch": 0.5171597633136095, + "grad_norm": 1.0365827647380519, + "kl": 0.031982421875, + "learning_rate": 4.828402366863905e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 874 + }, + { + "completion_length": 133.9921875, + "epoch": 0.5177514792899408, + "grad_norm": 0.7469780747409583, + "kl": 0.037109375, + "learning_rate": 4.822485207100592e-07, + "loss": 0.0015, + "reward": 0.171875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 875 + }, + { + "completion_length": 126.7265625, + "epoch": 0.5183431952662721, + "grad_norm": 2.878188780296037, + "kl": 0.038330078125, + "learning_rate": 4.816568047337278e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 876 + }, + { + "completion_length": 117.734375, + "epoch": 0.5189349112426036, + "grad_norm": 0.49151408672878727, + "kl": 0.039794921875, + "learning_rate": 4.810650887573965e-07, + "loss": 0.0016, + "reward": 0.109375, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 877 + }, + { + "completion_length": 123.390625, + "epoch": 0.5195266272189349, + "grad_norm": 0.5210781974908597, + "kl": 0.032470703125, + "learning_rate": 4.804733727810651e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 878 + }, + { + "completion_length": 124.421875, + "epoch": 0.5201183431952663, + "grad_norm": 0.3433073420875398, + "kl": 0.0238037109375, + "learning_rate": 4.798816568047337e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 879 + }, + { + "completion_length": 128.171875, + "epoch": 0.5207100591715976, + "grad_norm": 0.5460569962232031, + "kl": 0.0264892578125, + "learning_rate": 4.792899408284024e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 880 + }, + { + "completion_length": 121.6953125, + "epoch": 0.521301775147929, + "grad_norm": 0.7297338261556664, + "kl": 0.036376953125, + "learning_rate": 4.78698224852071e-07, + "loss": 0.0015, + "reward": 0.1796875, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 881 + }, + { + "completion_length": 127.234375, + "epoch": 0.5218934911242603, + "grad_norm": 0.9304038874761167, + "kl": 0.031494140625, + "learning_rate": 4.781065088757397e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 882 + }, + { + "completion_length": 121.5859375, + "epoch": 0.5224852071005918, + "grad_norm": 0.7631174977580534, + "kl": 0.0272216796875, + "learning_rate": 4.775147928994082e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 883 + }, + { + "completion_length": 129.25, + "epoch": 0.5230769230769231, + "grad_norm": 0.5456337476554487, + "kl": 0.0341796875, + "learning_rate": 4.769230769230769e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 884 + }, + { + "completion_length": 120.3671875, + "epoch": 0.5236686390532544, + "grad_norm": 0.45840141664753903, + "kl": 0.0228271484375, + "learning_rate": 4.7633136094674555e-07, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 885 + }, + { + "completion_length": 119.609375, + "epoch": 0.5242603550295858, + "grad_norm": 0.4865197701187135, + "kl": 0.033203125, + "learning_rate": 4.757396449704142e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 886 + }, + { + "completion_length": 131.2578125, + "epoch": 0.5248520710059171, + "grad_norm": 0.5509065272284585, + "kl": 0.02978515625, + "learning_rate": 4.751479289940828e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 887 + }, + { + "completion_length": 116.2578125, + "epoch": 0.5254437869822485, + "grad_norm": 0.45487568991304345, + "kl": 0.033935546875, + "learning_rate": 4.7455621301775144e-07, + "loss": 0.0014, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 888 + }, + { + "completion_length": 121.6875, + "epoch": 0.5260355029585799, + "grad_norm": 1.246688376815831, + "kl": 0.03759765625, + "learning_rate": 4.739644970414201e-07, + "loss": 0.0015, + "reward": 0.171875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 889 + }, + { + "completion_length": 130.234375, + "epoch": 0.5266272189349113, + "grad_norm": 0.2896627713826125, + "kl": 0.0272216796875, + "learning_rate": 4.733727810650887e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 890 + }, + { + "completion_length": 130.0625, + "epoch": 0.5272189349112426, + "grad_norm": 0.7502766464806969, + "kl": 0.0245361328125, + "learning_rate": 4.727810650887574e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 891 + }, + { + "completion_length": 131.9921875, + "epoch": 0.527810650887574, + "grad_norm": 0.7991825848656349, + "kl": 0.03369140625, + "learning_rate": 4.72189349112426e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 892 + }, + { + "completion_length": 132.8984375, + "epoch": 0.5284023668639053, + "grad_norm": 0.7203695906828556, + "kl": 0.024658203125, + "learning_rate": 4.7159763313609464e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 893 + }, + { + "completion_length": 142.0, + "epoch": 0.5289940828402366, + "grad_norm": 0.7620959269591525, + "kl": 0.0264892578125, + "learning_rate": 4.7100591715976327e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 894 + }, + { + "completion_length": 131.5234375, + "epoch": 0.5295857988165681, + "grad_norm": 0.4429056872621177, + "kl": 0.02880859375, + "learning_rate": 4.7041420118343195e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 895 + }, + { + "completion_length": 126.1875, + "epoch": 0.5301775147928994, + "grad_norm": 0.44250008354453646, + "kl": 0.030517578125, + "learning_rate": 4.6982248520710053e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 896 + }, + { + "completion_length": 138.8671875, + "epoch": 0.5307692307692308, + "grad_norm": 0.42488794895692206, + "kl": 0.022705078125, + "learning_rate": 4.692307692307692e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 897 + }, + { + "completion_length": 131.5078125, + "epoch": 0.5313609467455621, + "grad_norm": 0.10568584693113478, + "kl": 0.02587890625, + "learning_rate": 4.6863905325443784e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 898 + }, + { + "completion_length": 128.5390625, + "epoch": 0.5319526627218935, + "grad_norm": 3.1714437007668965, + "kl": 0.0308837890625, + "learning_rate": 4.680473372781065e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 899 + }, + { + "completion_length": 127.9609375, + "epoch": 0.5325443786982249, + "grad_norm": 0.3645039266844825, + "kl": 0.044921875, + "learning_rate": 4.674556213017751e-07, + "loss": 0.0018, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 900 + }, + { + "completion_length": 129.2265625, + "epoch": 0.5331360946745562, + "grad_norm": 0.45731376710063726, + "kl": 0.033203125, + "learning_rate": 4.668639053254438e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 901 + }, + { + "completion_length": 126.0703125, + "epoch": 0.5337278106508876, + "grad_norm": 0.6233842431411376, + "kl": 0.0262451171875, + "learning_rate": 4.6627218934911236e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 902 + }, + { + "completion_length": 127.5078125, + "epoch": 0.5343195266272189, + "grad_norm": 0.6300356047415272, + "kl": 0.0284423828125, + "learning_rate": 4.6568047337278105e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 903 + }, + { + "completion_length": 124.7890625, + "epoch": 0.5349112426035503, + "grad_norm": 0.4988337008736119, + "kl": 0.0281982421875, + "learning_rate": 4.6508875739644973e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 904 + }, + { + "completion_length": 121.7578125, + "epoch": 0.5355029585798816, + "grad_norm": 0.5135553769461116, + "kl": 0.02587890625, + "learning_rate": 4.644970414201183e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 905 + }, + { + "completion_length": 126.796875, + "epoch": 0.5360946745562131, + "grad_norm": 0.5931029798525287, + "kl": 0.0189208984375, + "learning_rate": 4.63905325443787e-07, + "loss": 0.0008, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 906 + }, + { + "completion_length": 125.875, + "epoch": 0.5366863905325444, + "grad_norm": 0.515898184096029, + "kl": 0.01513671875, + "learning_rate": 4.633136094674556e-07, + "loss": 0.0006, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 907 + }, + { + "completion_length": 119.8359375, + "epoch": 0.5372781065088758, + "grad_norm": 0.5670568997995135, + "kl": 0.0260009765625, + "learning_rate": 4.6272189349112425e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 908 + }, + { + "completion_length": 121.625, + "epoch": 0.5378698224852071, + "grad_norm": 0.49364934818636713, + "kl": 0.0191650390625, + "learning_rate": 4.621301775147929e-07, + "loss": 0.0008, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 909 + }, + { + "completion_length": 124.8671875, + "epoch": 0.5384615384615384, + "grad_norm": 0.4709812131059322, + "kl": 0.025390625, + "learning_rate": 4.6153846153846156e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 910 + }, + { + "completion_length": 119.25, + "epoch": 0.5390532544378698, + "grad_norm": 0.6532631674728641, + "kl": 0.0228271484375, + "learning_rate": 4.6094674556213014e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.0859375, + "step": 911 + }, + { + "completion_length": 108.3203125, + "epoch": 0.5396449704142012, + "grad_norm": 0.48548863142133014, + "kl": 0.04638671875, + "learning_rate": 4.603550295857988e-07, + "loss": 0.0019, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 912 + }, + { + "completion_length": 116.8828125, + "epoch": 0.5402366863905326, + "grad_norm": 0.40574007945702933, + "kl": 0.0255126953125, + "learning_rate": 4.5976331360946745e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 913 + }, + { + "completion_length": 105.1171875, + "epoch": 0.5408284023668639, + "grad_norm": 0.8071733127440862, + "kl": 0.032958984375, + "learning_rate": 4.591715976331361e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 914 + }, + { + "completion_length": 110.9296875, + "epoch": 0.5414201183431953, + "grad_norm": 0.565138717679982, + "kl": 0.03125, + "learning_rate": 4.585798816568047e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 915 + }, + { + "completion_length": 114.1484375, + "epoch": 0.5420118343195266, + "grad_norm": 1.2451339753053774, + "kl": 0.0361328125, + "learning_rate": 4.5798816568047334e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 916 + }, + { + "completion_length": 115.46875, + "epoch": 0.5426035502958579, + "grad_norm": 3.4455793190864377, + "kl": 0.028564453125, + "learning_rate": 4.5739644970414197e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 917 + }, + { + "completion_length": 115.078125, + "epoch": 0.5431952662721894, + "grad_norm": 0.3164772331951353, + "kl": 0.02587890625, + "learning_rate": 4.5680473372781066e-07, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 918 + }, + { + "completion_length": 113.5234375, + "epoch": 0.5437869822485207, + "grad_norm": 0.25893306534423455, + "kl": 0.0220947265625, + "learning_rate": 4.5621301775147923e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 919 + }, + { + "completion_length": 124.890625, + "epoch": 0.5443786982248521, + "grad_norm": 0.38111621017429825, + "kl": 0.0184326171875, + "learning_rate": 4.556213017751479e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 920 + }, + { + "completion_length": 114.171875, + "epoch": 0.5449704142011834, + "grad_norm": 0.6510214862926252, + "kl": 0.032470703125, + "learning_rate": 4.5502958579881655e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 921 + }, + { + "completion_length": 113.9375, + "epoch": 0.5455621301775148, + "grad_norm": 0.7089166646394723, + "kl": 0.034912109375, + "learning_rate": 4.544378698224852e-07, + "loss": 0.0014, + "reward": 0.1171875, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 922 + }, + { + "completion_length": 105.078125, + "epoch": 0.5461538461538461, + "grad_norm": 0.7098607248190701, + "kl": 0.0302734375, + "learning_rate": 4.538461538461538e-07, + "loss": 0.0012, + "reward": 0.1875, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 923 + }, + { + "completion_length": 125.4453125, + "epoch": 0.5467455621301776, + "grad_norm": 0.5157239785596723, + "kl": 0.0267333984375, + "learning_rate": 4.532544378698225e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 924 + }, + { + "completion_length": 113.859375, + "epoch": 0.5473372781065089, + "grad_norm": 0.5158085958756266, + "kl": 0.0291748046875, + "learning_rate": 4.5266272189349107e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 925 + }, + { + "completion_length": 114.3515625, + "epoch": 0.5479289940828402, + "grad_norm": 0.6157688515763056, + "kl": 0.02392578125, + "learning_rate": 4.5207100591715975e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 926 + }, + { + "completion_length": 118.7109375, + "epoch": 0.5485207100591716, + "grad_norm": 0.5050682514264708, + "kl": 0.024658203125, + "learning_rate": 4.514792899408284e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 927 + }, + { + "completion_length": 115.6328125, + "epoch": 0.5491124260355029, + "grad_norm": 0.8131566554670643, + "kl": 0.04052734375, + "learning_rate": 4.50887573964497e-07, + "loss": 0.0016, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 928 + }, + { + "completion_length": 115.5078125, + "epoch": 0.5497041420118344, + "grad_norm": 0.5731904251019695, + "kl": 0.0224609375, + "learning_rate": 4.5029585798816564e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 929 + }, + { + "completion_length": 128.703125, + "epoch": 0.5502958579881657, + "grad_norm": 0.4509459489616563, + "kl": 0.025634765625, + "learning_rate": 4.497041420118343e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 930 + }, + { + "completion_length": 127.5390625, + "epoch": 0.5508875739644971, + "grad_norm": 0.07558037802374964, + "kl": 0.0211181640625, + "learning_rate": 4.491124260355029e-07, + "loss": 0.0008, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 931 + }, + { + "completion_length": 124.1875, + "epoch": 0.5514792899408284, + "grad_norm": 0.40845982524739816, + "kl": 0.0291748046875, + "learning_rate": 4.485207100591716e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 932 + }, + { + "completion_length": 118.0546875, + "epoch": 0.5520710059171597, + "grad_norm": 0.9604621903528362, + "kl": 0.043212890625, + "learning_rate": 4.4792899408284026e-07, + "loss": 0.0017, + "reward": 0.15625, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 933 + }, + { + "completion_length": 115.265625, + "epoch": 0.5526627218934911, + "grad_norm": 0.7154285424009027, + "kl": 0.035400390625, + "learning_rate": 4.4733727810650884e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 934 + }, + { + "completion_length": 136.4296875, + "epoch": 0.5532544378698225, + "grad_norm": 0.12780982583459113, + "kl": 0.026611328125, + "learning_rate": 4.467455621301775e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 935 + }, + { + "completion_length": 119.59375, + "epoch": 0.5538461538461539, + "grad_norm": 1.3489718926008183, + "kl": 0.038330078125, + "learning_rate": 4.4615384615384615e-07, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.11904378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 936 + }, + { + "completion_length": 123.4453125, + "epoch": 0.5544378698224852, + "grad_norm": 0.5943635998492177, + "kl": 0.025146484375, + "learning_rate": 4.455621301775148e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 937 + }, + { + "completion_length": 123.546875, + "epoch": 0.5550295857988166, + "grad_norm": 1.1145449537357857, + "kl": 0.03466796875, + "learning_rate": 4.449704142011834e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 938 + }, + { + "completion_length": 131.1640625, + "epoch": 0.5556213017751479, + "grad_norm": 0.5748752374611024, + "kl": 0.0263671875, + "learning_rate": 4.443786982248521e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 939 + }, + { + "completion_length": 124.8515625, + "epoch": 0.5562130177514792, + "grad_norm": 0.733398222906948, + "kl": 0.039794921875, + "learning_rate": 4.437869822485207e-07, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 940 + }, + { + "completion_length": 126.5546875, + "epoch": 0.5568047337278107, + "grad_norm": 0.5646623244896979, + "kl": 0.031005859375, + "learning_rate": 4.4319526627218936e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 941 + }, + { + "completion_length": 128.5234375, + "epoch": 0.557396449704142, + "grad_norm": 1.0040528167936542, + "kl": 0.02587890625, + "learning_rate": 4.4260355029585793e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 942 + }, + { + "completion_length": 121.3359375, + "epoch": 0.5579881656804734, + "grad_norm": 0.13332355367171583, + "kl": 0.0244140625, + "learning_rate": 4.420118343195266e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 943 + }, + { + "completion_length": 119.09375, + "epoch": 0.5585798816568047, + "grad_norm": 0.306116143931757, + "kl": 0.018310546875, + "learning_rate": 4.4142011834319525e-07, + "loss": 0.0007, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 944 + }, + { + "completion_length": 121.671875, + "epoch": 0.5591715976331361, + "grad_norm": 0.3394072782883282, + "kl": 0.0255126953125, + "learning_rate": 4.408284023668639e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 945 + }, + { + "completion_length": 114.9921875, + "epoch": 0.5597633136094674, + "grad_norm": 0.7382211809489646, + "kl": 0.0439453125, + "learning_rate": 4.402366863905325e-07, + "loss": 0.0018, + "reward": 0.140625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 946 + }, + { + "completion_length": 121.953125, + "epoch": 0.5603550295857989, + "grad_norm": 0.5059789321776144, + "kl": 0.0262451171875, + "learning_rate": 4.396449704142012e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 947 + }, + { + "completion_length": 123.609375, + "epoch": 0.5609467455621302, + "grad_norm": 0.6302312716443117, + "kl": 0.031982421875, + "learning_rate": 4.3905325443786977e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 948 + }, + { + "completion_length": 123.9375, + "epoch": 0.5615384615384615, + "grad_norm": 0.06783600290407074, + "kl": 0.020263671875, + "learning_rate": 4.3846153846153845e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 949 + }, + { + "completion_length": 120.421875, + "epoch": 0.5621301775147929, + "grad_norm": 0.31619029182544944, + "kl": 0.021240234375, + "learning_rate": 4.378698224852071e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 950 + }, + { + "completion_length": 126.5078125, + "epoch": 0.5627218934911242, + "grad_norm": 0.13101099004021421, + "kl": 0.024658203125, + "learning_rate": 4.372781065088757e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 951 + }, + { + "completion_length": 124.015625, + "epoch": 0.5633136094674556, + "grad_norm": 0.43880037294524976, + "kl": 0.032470703125, + "learning_rate": 4.3668639053254434e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 952 + }, + { + "completion_length": 127.296875, + "epoch": 0.563905325443787, + "grad_norm": 0.5927204889399519, + "kl": 0.03759765625, + "learning_rate": 4.36094674556213e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 953 + }, + { + "completion_length": 123.2421875, + "epoch": 0.5644970414201184, + "grad_norm": 0.7806248417432643, + "kl": 0.02880859375, + "learning_rate": 4.355029585798816e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 954 + }, + { + "completion_length": 121.0390625, + "epoch": 0.5650887573964497, + "grad_norm": 1.721131376230108, + "kl": 0.034423828125, + "learning_rate": 4.349112426035503e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 955 + }, + { + "completion_length": 129.0546875, + "epoch": 0.565680473372781, + "grad_norm": 0.6395396332839758, + "kl": 0.03515625, + "learning_rate": 4.343195266272189e-07, + "loss": 0.0014, + "reward": 0.1484375, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 956 + }, + { + "completion_length": 129.8828125, + "epoch": 0.5662721893491124, + "grad_norm": 0.4434484438454899, + "kl": 0.0220947265625, + "learning_rate": 4.3372781065088754e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 957 + }, + { + "completion_length": 125.9140625, + "epoch": 0.5668639053254438, + "grad_norm": 0.26363910515438205, + "kl": 0.01904296875, + "learning_rate": 4.3313609467455617e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 958 + }, + { + "completion_length": 118.5703125, + "epoch": 0.5674556213017752, + "grad_norm": 0.5007548406720777, + "kl": 0.0244140625, + "learning_rate": 4.3254437869822486e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 959 + }, + { + "completion_length": 117.6796875, + "epoch": 0.5680473372781065, + "grad_norm": 0.4247739350724073, + "kl": 0.0205078125, + "learning_rate": 4.3195266272189343e-07, + "loss": 0.0008, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 960 + }, + { + "completion_length": 122.765625, + "epoch": 0.5686390532544379, + "grad_norm": 0.6200811052962872, + "kl": 0.0419921875, + "learning_rate": 4.313609467455621e-07, + "loss": 0.0017, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 961 + }, + { + "completion_length": 120.65625, + "epoch": 0.5692307692307692, + "grad_norm": 0.2887711021794561, + "kl": 0.0225830078125, + "learning_rate": 4.307692307692308e-07, + "loss": 0.0009, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 962 + }, + { + "completion_length": 131.1640625, + "epoch": 0.5698224852071005, + "grad_norm": 0.4751725799321403, + "kl": 0.020263671875, + "learning_rate": 4.301775147928994e-07, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 963 + }, + { + "completion_length": 113.6640625, + "epoch": 0.570414201183432, + "grad_norm": 0.6148117752442909, + "kl": 0.0283203125, + "learning_rate": 4.2958579881656806e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 964 + }, + { + "completion_length": 110.7265625, + "epoch": 0.5710059171597633, + "grad_norm": 0.8953915121207694, + "kl": 0.03369140625, + "learning_rate": 4.289940828402367e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 965 + }, + { + "completion_length": 126.09375, + "epoch": 0.5715976331360947, + "grad_norm": 0.10707665361139143, + "kl": 0.032470703125, + "learning_rate": 4.284023668639053e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 966 + }, + { + "completion_length": 117.5, + "epoch": 0.572189349112426, + "grad_norm": 0.4932032381923405, + "kl": 0.027099609375, + "learning_rate": 4.2781065088757395e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 967 + }, + { + "completion_length": 123.0859375, + "epoch": 0.5727810650887574, + "grad_norm": 0.25259320936091345, + "kl": 0.0213623046875, + "learning_rate": 4.272189349112426e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 968 + }, + { + "completion_length": 119.9609375, + "epoch": 0.5733727810650887, + "grad_norm": 0.45530864670538707, + "kl": 0.025634765625, + "learning_rate": 4.266272189349112e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 969 + }, + { + "completion_length": 120.34375, + "epoch": 0.5739644970414202, + "grad_norm": 0.23470961871706433, + "kl": 0.0211181640625, + "learning_rate": 4.260355029585799e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 970 + }, + { + "completion_length": 125.546875, + "epoch": 0.5745562130177515, + "grad_norm": 0.5915254896999852, + "kl": 0.0296630859375, + "learning_rate": 4.2544378698224847e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 971 + }, + { + "completion_length": 130.09375, + "epoch": 0.5751479289940828, + "grad_norm": 0.5260854324297701, + "kl": 0.0244140625, + "learning_rate": 4.2485207100591715e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 972 + }, + { + "completion_length": 121.3359375, + "epoch": 0.5757396449704142, + "grad_norm": 0.5575580951986822, + "kl": 0.02685546875, + "learning_rate": 4.242603550295858e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 973 + }, + { + "completion_length": 130.125, + "epoch": 0.5763313609467455, + "grad_norm": 1.1177775981554794, + "kl": 0.02783203125, + "learning_rate": 4.236686390532544e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 974 + }, + { + "completion_length": 119.9453125, + "epoch": 0.5769230769230769, + "grad_norm": 1.0162106502319836, + "kl": 0.03466796875, + "learning_rate": 4.2307692307692304e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 975 + }, + { + "completion_length": 119.265625, + "epoch": 0.5775147928994083, + "grad_norm": 15.637433465362982, + "kl": 0.034912109375, + "learning_rate": 4.224852071005917e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 976 + }, + { + "completion_length": 121.484375, + "epoch": 0.5781065088757397, + "grad_norm": 0.788244299671909, + "kl": 0.032958984375, + "learning_rate": 4.218934911242603e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 977 + }, + { + "completion_length": 124.6953125, + "epoch": 0.578698224852071, + "grad_norm": 0.566285720039258, + "kl": 0.0267333984375, + "learning_rate": 4.21301775147929e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 978 + }, + { + "completion_length": 128.59375, + "epoch": 0.5792899408284023, + "grad_norm": 0.34709274471086143, + "kl": 0.022216796875, + "learning_rate": 4.207100591715976e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 979 + }, + { + "completion_length": 116.8203125, + "epoch": 0.5798816568047337, + "grad_norm": 0.528347422885083, + "kl": 0.0242919921875, + "learning_rate": 4.2011834319526624e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 980 + }, + { + "completion_length": 122.1875, + "epoch": 0.580473372781065, + "grad_norm": 1.1936796731276804, + "kl": 0.0255126953125, + "learning_rate": 4.195266272189349e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 981 + }, + { + "completion_length": 106.4765625, + "epoch": 0.5810650887573965, + "grad_norm": 0.4969574089718798, + "kl": 0.03662109375, + "learning_rate": 4.1893491124260356e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 982 + }, + { + "completion_length": 119.5234375, + "epoch": 0.5816568047337278, + "grad_norm": 0.7675548548273173, + "kl": 0.033447265625, + "learning_rate": 4.1834319526627213e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 983 + }, + { + "completion_length": 119.796875, + "epoch": 0.5822485207100592, + "grad_norm": 0.47348416272758076, + "kl": 0.03173828125, + "learning_rate": 4.177514792899408e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 984 + }, + { + "completion_length": 127.5078125, + "epoch": 0.5828402366863905, + "grad_norm": 0.7127225714338157, + "kl": 0.0361328125, + "learning_rate": 4.1715976331360945e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 985 + }, + { + "completion_length": 116.0390625, + "epoch": 0.5834319526627219, + "grad_norm": 1.0388589486295385, + "kl": 0.03466796875, + "learning_rate": 4.165680473372781e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 986 + }, + { + "completion_length": 115.03125, + "epoch": 0.5840236686390533, + "grad_norm": 0.9326837823217694, + "kl": 0.03759765625, + "learning_rate": 4.159763313609467e-07, + "loss": 0.0015, + "reward": 0.1796875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 987 + }, + { + "completion_length": 117.484375, + "epoch": 0.5846153846153846, + "grad_norm": 0.43533485237551195, + "kl": 0.0184326171875, + "learning_rate": 4.153846153846154e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 988 + }, + { + "completion_length": 122.84375, + "epoch": 0.585207100591716, + "grad_norm": 0.27861182505082754, + "kl": 0.022705078125, + "learning_rate": 4.1479289940828397e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 989 + }, + { + "completion_length": 121.71875, + "epoch": 0.5857988165680473, + "grad_norm": 0.5748075722437701, + "kl": 0.03564453125, + "learning_rate": 4.1420118343195265e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 990 + }, + { + "completion_length": 119.3046875, + "epoch": 0.5863905325443787, + "grad_norm": 0.6166119122264548, + "kl": 0.024169921875, + "learning_rate": 4.1360946745562133e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 991 + }, + { + "completion_length": 122.9921875, + "epoch": 0.58698224852071, + "grad_norm": 1.095711555855484, + "kl": 0.02685546875, + "learning_rate": 4.130177514792899e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 992 + }, + { + "completion_length": 121.421875, + "epoch": 0.5875739644970415, + "grad_norm": 0.051712625816338835, + "kl": 0.01495361328125, + "learning_rate": 4.124260355029586e-07, + "loss": 0.0006, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 993 + }, + { + "completion_length": 121.5859375, + "epoch": 0.5881656804733728, + "grad_norm": 1.572238343267962, + "kl": 0.0264892578125, + "learning_rate": 4.118343195266272e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 994 + }, + { + "completion_length": 125.3203125, + "epoch": 0.5887573964497042, + "grad_norm": 0.3889786940061513, + "kl": 0.0238037109375, + "learning_rate": 4.1124260355029585e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 995 + }, + { + "completion_length": 126.609375, + "epoch": 0.5893491124260355, + "grad_norm": 0.7571873856910055, + "kl": 0.0255126953125, + "learning_rate": 4.106508875739645e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 996 + }, + { + "completion_length": 123.953125, + "epoch": 0.5899408284023668, + "grad_norm": 0.445147128510999, + "kl": 0.0244140625, + "learning_rate": 4.100591715976331e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 997 + }, + { + "completion_length": 110.453125, + "epoch": 0.5905325443786982, + "grad_norm": 0.6905979658461715, + "kl": 0.04052734375, + "learning_rate": 4.0946745562130174e-07, + "loss": 0.0016, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 998 + }, + { + "completion_length": 124.328125, + "epoch": 0.5911242603550296, + "grad_norm": 0.55089419521903, + "kl": 0.0263671875, + "learning_rate": 4.088757396449704e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 999 + }, + { + "completion_length": 121.265625, + "epoch": 0.591715976331361, + "grad_norm": 0.4465988093308378, + "kl": 0.03173828125, + "learning_rate": 4.08284023668639e-07, + "loss": 0.0013, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1000 + }, + { + "completion_length": 118.21875, + "epoch": 0.5923076923076923, + "grad_norm": 0.4978929345927383, + "kl": 0.024169921875, + "learning_rate": 4.076923076923077e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1001 + }, + { + "completion_length": 111.5703125, + "epoch": 0.5928994082840237, + "grad_norm": 0.7881958405945094, + "kl": 0.029052734375, + "learning_rate": 4.071005917159763e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1002 + }, + { + "completion_length": 115.21875, + "epoch": 0.593491124260355, + "grad_norm": 0.420031425613086, + "kl": 0.03271484375, + "learning_rate": 4.0650887573964495e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1003 + }, + { + "completion_length": 121.078125, + "epoch": 0.5940828402366863, + "grad_norm": 0.5269980584202366, + "kl": 0.02734375, + "learning_rate": 4.059171597633136e-07, + "loss": 0.0011, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1004 + }, + { + "completion_length": 117.8125, + "epoch": 0.5946745562130178, + "grad_norm": 0.5547753008589887, + "kl": 0.016357421875, + "learning_rate": 4.0532544378698226e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1005 + }, + { + "completion_length": 121.8984375, + "epoch": 0.5952662721893491, + "grad_norm": 0.33287100957148724, + "kl": 0.0264892578125, + "learning_rate": 4.0473372781065084e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1006 + }, + { + "completion_length": 116.3125, + "epoch": 0.5958579881656805, + "grad_norm": 0.8718671885081928, + "kl": 0.0220947265625, + "learning_rate": 4.041420118343195e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1007 + }, + { + "completion_length": 119.7421875, + "epoch": 0.5964497041420118, + "grad_norm": 0.9161381062659466, + "kl": 0.0220947265625, + "learning_rate": 4.0355029585798815e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1008 + }, + { + "completion_length": 127.796875, + "epoch": 0.5970414201183432, + "grad_norm": 0.3070264981886789, + "kl": 0.03076171875, + "learning_rate": 4.029585798816568e-07, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1009 + }, + { + "completion_length": 135.625, + "epoch": 0.5976331360946746, + "grad_norm": 0.5643480894006191, + "kl": 0.0302734375, + "learning_rate": 4.023668639053254e-07, + "loss": 0.0012, + "reward": 0.15625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1010 + }, + { + "completion_length": 128.984375, + "epoch": 0.598224852071006, + "grad_norm": 0.06728562708033776, + "kl": 0.022216796875, + "learning_rate": 4.017751479289941e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1011 + }, + { + "completion_length": 129.1953125, + "epoch": 0.5988165680473373, + "grad_norm": 0.247266504569265, + "kl": 0.0224609375, + "learning_rate": 4.0118343195266267e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1012 + }, + { + "completion_length": 122.7421875, + "epoch": 0.5994082840236686, + "grad_norm": 0.38411596978716606, + "kl": 0.031982421875, + "learning_rate": 4.0059171597633135e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1013 + }, + { + "completion_length": 130.3671875, + "epoch": 0.6, + "grad_norm": 1.5839666638531515, + "kl": 0.0283203125, + "learning_rate": 4e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1014 + }, + { + "completion_length": 124.6640625, + "epoch": 0.6005917159763313, + "grad_norm": 0.4181026070661975, + "kl": 0.0281982421875, + "learning_rate": 3.994082840236686e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1015 + }, + { + "completion_length": 120.625, + "epoch": 0.6011834319526628, + "grad_norm": 0.43347978382685237, + "kl": 0.036865234375, + "learning_rate": 3.9881656804733724e-07, + "loss": 0.0015, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1016 + }, + { + "completion_length": 121.859375, + "epoch": 0.6017751479289941, + "grad_norm": 0.7189557133197073, + "kl": 0.03662109375, + "learning_rate": 3.982248520710059e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1017 + }, + { + "completion_length": 123.09375, + "epoch": 0.6023668639053255, + "grad_norm": 0.6910716742042927, + "kl": 0.030029296875, + "learning_rate": 3.976331360946745e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1018 + }, + { + "completion_length": 118.0546875, + "epoch": 0.6029585798816568, + "grad_norm": 0.4056114947791359, + "kl": 0.0244140625, + "learning_rate": 3.970414201183432e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1019 + }, + { + "completion_length": 121.2734375, + "epoch": 0.6035502958579881, + "grad_norm": 0.5330220861424653, + "kl": 0.032958984375, + "learning_rate": 3.9644970414201187e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1020 + }, + { + "completion_length": 129.703125, + "epoch": 0.6041420118343195, + "grad_norm": 0.6385283394213404, + "kl": 0.0247802734375, + "learning_rate": 3.9585798816568044e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1021 + }, + { + "completion_length": 123.046875, + "epoch": 0.6047337278106509, + "grad_norm": 0.457587657228225, + "kl": 0.0341796875, + "learning_rate": 3.9526627218934913e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1022 + }, + { + "completion_length": 126.6328125, + "epoch": 0.6053254437869823, + "grad_norm": 0.7333308393413689, + "kl": 0.039306640625, + "learning_rate": 3.946745562130177e-07, + "loss": 0.0016, + "reward": 0.1640625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1023 + }, + { + "completion_length": 125.6484375, + "epoch": 0.6059171597633136, + "grad_norm": 0.550473020590258, + "kl": 0.02783203125, + "learning_rate": 3.940828402366864e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1024 + }, + { + "completion_length": 114.828125, + "epoch": 0.606508875739645, + "grad_norm": 0.6005003100874552, + "kl": 0.034423828125, + "learning_rate": 3.93491124260355e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1025 + }, + { + "completion_length": 125.8359375, + "epoch": 0.6071005917159763, + "grad_norm": 0.6342699593595745, + "kl": 0.02392578125, + "learning_rate": 3.9289940828402365e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1026 + }, + { + "completion_length": 124.9765625, + "epoch": 0.6076923076923076, + "grad_norm": 0.3590467736283468, + "kl": 0.0262451171875, + "learning_rate": 3.923076923076923e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1027 + }, + { + "completion_length": 125.8125, + "epoch": 0.6082840236686391, + "grad_norm": 0.4652014077335686, + "kl": 0.03076171875, + "learning_rate": 3.9171597633136096e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1028 + }, + { + "completion_length": 117.8984375, + "epoch": 0.6088757396449704, + "grad_norm": 0.5514857249520951, + "kl": 0.0247802734375, + "learning_rate": 3.9112426035502954e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1029 + }, + { + "completion_length": 129.6171875, + "epoch": 0.6094674556213018, + "grad_norm": 0.36606844305730535, + "kl": 0.0294189453125, + "learning_rate": 3.905325443786982e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1030 + }, + { + "completion_length": 115.125, + "epoch": 0.6100591715976331, + "grad_norm": 0.3821828890444864, + "kl": 0.026611328125, + "learning_rate": 3.8994082840236685e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1031 + }, + { + "completion_length": 114.71875, + "epoch": 0.6106508875739645, + "grad_norm": 0.59712441836101, + "kl": 0.0257568359375, + "learning_rate": 3.893491124260355e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1032 + }, + { + "completion_length": 117.421875, + "epoch": 0.6112426035502958, + "grad_norm": 0.5313531555533063, + "kl": 0.03125, + "learning_rate": 3.887573964497041e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1033 + }, + { + "completion_length": 126.234375, + "epoch": 0.6118343195266273, + "grad_norm": 0.2672565814612442, + "kl": 0.025390625, + "learning_rate": 3.881656804733728e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1034 + }, + { + "completion_length": 126.515625, + "epoch": 0.6124260355029586, + "grad_norm": 0.4817962832343136, + "kl": 0.0284423828125, + "learning_rate": 3.8757396449704137e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1035 + }, + { + "completion_length": 129.90625, + "epoch": 0.6130177514792899, + "grad_norm": 0.8901835036381095, + "kl": 0.02587890625, + "learning_rate": 3.8698224852071005e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1036 + }, + { + "completion_length": 126.546875, + "epoch": 0.6136094674556213, + "grad_norm": 0.4927793952106921, + "kl": 0.03271484375, + "learning_rate": 3.863905325443787e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1037 + }, + { + "completion_length": 120.0546875, + "epoch": 0.6142011834319526, + "grad_norm": 0.5063044594753798, + "kl": 0.018798828125, + "learning_rate": 3.857988165680473e-07, + "loss": 0.0008, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1038 + }, + { + "completion_length": 122.40625, + "epoch": 0.6147928994082841, + "grad_norm": 0.32319197828705254, + "kl": 0.021728515625, + "learning_rate": 3.8520710059171594e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1039 + }, + { + "completion_length": 120.7421875, + "epoch": 0.6153846153846154, + "grad_norm": 0.34841826433069256, + "kl": 0.02734375, + "learning_rate": 3.8461538461538463e-07, + "loss": 0.0011, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1040 + }, + { + "completion_length": 121.1640625, + "epoch": 0.6159763313609468, + "grad_norm": 0.7196969763603306, + "kl": 0.0260009765625, + "learning_rate": 3.840236686390532e-07, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1041 + }, + { + "completion_length": 121.7265625, + "epoch": 0.6165680473372781, + "grad_norm": 0.6627111491347547, + "kl": 0.029296875, + "learning_rate": 3.834319526627219e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1042 + }, + { + "completion_length": 110.0859375, + "epoch": 0.6171597633136094, + "grad_norm": 0.5151884676197164, + "kl": 0.0341796875, + "learning_rate": 3.828402366863905e-07, + "loss": 0.0014, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1043 + }, + { + "completion_length": 123.4921875, + "epoch": 0.6177514792899408, + "grad_norm": 0.4316443173420594, + "kl": 0.021728515625, + "learning_rate": 3.8224852071005915e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1044 + }, + { + "completion_length": 119.75, + "epoch": 0.6183431952662722, + "grad_norm": 0.055306372971866094, + "kl": 0.024169921875, + "learning_rate": 3.816568047337278e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1045 + }, + { + "completion_length": 121.65625, + "epoch": 0.6189349112426036, + "grad_norm": 0.4865218391658723, + "kl": 0.030517578125, + "learning_rate": 3.8106508875739646e-07, + "loss": 0.0012, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1046 + }, + { + "completion_length": 103.5859375, + "epoch": 0.6195266272189349, + "grad_norm": 0.3607433637009951, + "kl": 0.021484375, + "learning_rate": 3.8047337278106504e-07, + "loss": 0.0009, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1047 + }, + { + "completion_length": 115.140625, + "epoch": 0.6201183431952663, + "grad_norm": 0.8902626156344804, + "kl": 0.03173828125, + "learning_rate": 3.798816568047337e-07, + "loss": 0.0013, + "reward": 0.1953125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1048 + }, + { + "completion_length": 117.5703125, + "epoch": 0.6207100591715976, + "grad_norm": 0.4134049019202975, + "kl": 0.024169921875, + "learning_rate": 3.792899408284024e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1049 + }, + { + "completion_length": 120.296875, + "epoch": 0.621301775147929, + "grad_norm": 0.06036597896647533, + "kl": 0.0184326171875, + "learning_rate": 3.78698224852071e-07, + "loss": 0.0007, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1050 + }, + { + "completion_length": 120.9375, + "epoch": 0.6218934911242604, + "grad_norm": 0.622524029873957, + "kl": 0.0341796875, + "learning_rate": 3.7810650887573966e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1051 + }, + { + "completion_length": 121.96875, + "epoch": 0.6224852071005917, + "grad_norm": 0.6357873723054446, + "kl": 0.02490234375, + "learning_rate": 3.7751479289940824e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1052 + }, + { + "completion_length": 107.2265625, + "epoch": 0.6230769230769231, + "grad_norm": 0.6113275124608821, + "kl": 0.03369140625, + "learning_rate": 3.769230769230769e-07, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1053 + }, + { + "completion_length": 122.40625, + "epoch": 0.6236686390532544, + "grad_norm": 0.6419360369065918, + "kl": 0.0302734375, + "learning_rate": 3.7633136094674555e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1054 + }, + { + "completion_length": 122.9296875, + "epoch": 0.6242603550295858, + "grad_norm": 0.3501611776181825, + "kl": 0.024169921875, + "learning_rate": 3.757396449704142e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1055 + }, + { + "completion_length": 116.125, + "epoch": 0.6248520710059171, + "grad_norm": 0.5511262894276723, + "kl": 0.029296875, + "learning_rate": 3.751479289940828e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1056 + }, + { + "completion_length": 130.1015625, + "epoch": 0.6254437869822486, + "grad_norm": 1.1180366480656088, + "kl": 0.029052734375, + "learning_rate": 3.745562130177515e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1057 + }, + { + "completion_length": 118.046875, + "epoch": 0.6260355029585799, + "grad_norm": 0.7726219917962852, + "kl": 0.0294189453125, + "learning_rate": 3.7396449704142007e-07, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.11420939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1058 + }, + { + "completion_length": 112.09375, + "epoch": 0.6266272189349112, + "grad_norm": 1.2020952703081458, + "kl": 0.04052734375, + "learning_rate": 3.7337278106508876e-07, + "loss": 0.0016, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1059 + }, + { + "completion_length": 124.28125, + "epoch": 0.6272189349112426, + "grad_norm": 0.7615424288228301, + "kl": 0.029052734375, + "learning_rate": 3.727810650887574e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1060 + }, + { + "completion_length": 122.0390625, + "epoch": 0.6278106508875739, + "grad_norm": 0.41616742912188714, + "kl": 0.029296875, + "learning_rate": 3.72189349112426e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1061 + }, + { + "completion_length": 119.328125, + "epoch": 0.6284023668639053, + "grad_norm": 0.08459350947574343, + "kl": 0.0283203125, + "learning_rate": 3.7159763313609465e-07, + "loss": 0.0011, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1062 + }, + { + "completion_length": 113.1171875, + "epoch": 0.6289940828402367, + "grad_norm": 0.5308954946982305, + "kl": 0.026123046875, + "learning_rate": 3.7100591715976333e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.046875, + "step": 1063 + }, + { + "completion_length": 121.6796875, + "epoch": 0.6295857988165681, + "grad_norm": 0.4187313800990895, + "kl": 0.0283203125, + "learning_rate": 3.704142011834319e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1064 + }, + { + "completion_length": 113.1796875, + "epoch": 0.6301775147928994, + "grad_norm": 0.4302691646625525, + "kl": 0.02685546875, + "learning_rate": 3.698224852071006e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1065 + }, + { + "completion_length": 120.6796875, + "epoch": 0.6307692307692307, + "grad_norm": 0.43588019134539047, + "kl": 0.0223388671875, + "learning_rate": 3.692307692307692e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1066 + }, + { + "completion_length": 124.8046875, + "epoch": 0.6313609467455621, + "grad_norm": 0.21774791440246885, + "kl": 0.018798828125, + "learning_rate": 3.6863905325443785e-07, + "loss": 0.0008, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 1067 + }, + { + "completion_length": 113.0078125, + "epoch": 0.6319526627218935, + "grad_norm": 0.609154934853181, + "kl": 0.03564453125, + "learning_rate": 3.680473372781065e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1068 + }, + { + "completion_length": 119.984375, + "epoch": 0.6325443786982249, + "grad_norm": 0.47845165873626644, + "kl": 0.03125, + "learning_rate": 3.6745562130177516e-07, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1069 + }, + { + "completion_length": 124.8984375, + "epoch": 0.6331360946745562, + "grad_norm": 0.44422360886110396, + "kl": 0.023193359375, + "learning_rate": 3.6686390532544374e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1070 + }, + { + "completion_length": 113.6171875, + "epoch": 0.6337278106508876, + "grad_norm": 1.0471403733016325, + "kl": 0.034423828125, + "learning_rate": 3.662721893491124e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1071 + }, + { + "completion_length": 116.28125, + "epoch": 0.6343195266272189, + "grad_norm": 0.5680109952684057, + "kl": 0.026123046875, + "learning_rate": 3.6568047337278105e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1072 + }, + { + "completion_length": 118.40625, + "epoch": 0.6349112426035503, + "grad_norm": 0.5357891285065849, + "kl": 0.022705078125, + "learning_rate": 3.650887573964497e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1073 + }, + { + "completion_length": 115.078125, + "epoch": 0.6355029585798817, + "grad_norm": 0.35134101406881957, + "kl": 0.0262451171875, + "learning_rate": 3.644970414201183e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1074 + }, + { + "completion_length": 124.9296875, + "epoch": 0.636094674556213, + "grad_norm": 0.45085650886509254, + "kl": 0.025146484375, + "learning_rate": 3.63905325443787e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1075 + }, + { + "completion_length": 121.375, + "epoch": 0.6366863905325444, + "grad_norm": 0.4356091918168547, + "kl": 0.0267333984375, + "learning_rate": 3.6331360946745557e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1076 + }, + { + "completion_length": 109.921875, + "epoch": 0.6372781065088757, + "grad_norm": 0.36561176863584577, + "kl": 0.047119140625, + "learning_rate": 3.6272189349112425e-07, + "loss": 0.0019, + "reward": 0.1796875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 1077 + }, + { + "completion_length": 119.046875, + "epoch": 0.6378698224852071, + "grad_norm": 0.37071418458186844, + "kl": 0.026123046875, + "learning_rate": 3.6213017751479283e-07, + "loss": 0.001, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1078 + }, + { + "completion_length": 122.40625, + "epoch": 0.6384615384615384, + "grad_norm": 0.23246260591652426, + "kl": 0.0196533203125, + "learning_rate": 3.615384615384615e-07, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1079 + }, + { + "completion_length": 113.390625, + "epoch": 0.6390532544378699, + "grad_norm": 0.7671443110956332, + "kl": 0.0361328125, + "learning_rate": 3.609467455621302e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1080 + }, + { + "completion_length": 120.65625, + "epoch": 0.6396449704142012, + "grad_norm": 0.7475095322025289, + "kl": 0.03125, + "learning_rate": 3.603550295857988e-07, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1081 + }, + { + "completion_length": 118.265625, + "epoch": 0.6402366863905326, + "grad_norm": 2.733568485204059, + "kl": 0.029052734375, + "learning_rate": 3.5976331360946746e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1082 + }, + { + "completion_length": 119.1015625, + "epoch": 0.6408284023668639, + "grad_norm": 0.43398444193262925, + "kl": 0.025634765625, + "learning_rate": 3.591715976331361e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1083 + }, + { + "completion_length": 124.2265625, + "epoch": 0.6414201183431952, + "grad_norm": 0.6983019871530295, + "kl": 0.024658203125, + "learning_rate": 3.585798816568047e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1084 + }, + { + "completion_length": 118.9765625, + "epoch": 0.6420118343195266, + "grad_norm": 1.0822539694972522, + "kl": 0.029541015625, + "learning_rate": 3.5798816568047335e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1085 + }, + { + "completion_length": 123.765625, + "epoch": 0.642603550295858, + "grad_norm": 0.5249296477953708, + "kl": 0.037109375, + "learning_rate": 3.5739644970414203e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1086 + }, + { + "completion_length": 124.0078125, + "epoch": 0.6431952662721894, + "grad_norm": 0.6649413201447568, + "kl": 0.020263671875, + "learning_rate": 3.568047337278106e-07, + "loss": 0.0008, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1087 + }, + { + "completion_length": 117.0703125, + "epoch": 0.6437869822485207, + "grad_norm": 0.4235777255602693, + "kl": 0.03173828125, + "learning_rate": 3.562130177514793e-07, + "loss": 0.0013, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1088 + }, + { + "completion_length": 120.4765625, + "epoch": 0.644378698224852, + "grad_norm": 1.0559535843883394, + "kl": 0.048583984375, + "learning_rate": 3.556213017751479e-07, + "loss": 0.0019, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1089 + }, + { + "completion_length": 122.6875, + "epoch": 0.6449704142011834, + "grad_norm": 0.5931716512808286, + "kl": 0.0284423828125, + "learning_rate": 3.5502958579881655e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1090 + }, + { + "completion_length": 121.671875, + "epoch": 0.6455621301775148, + "grad_norm": 0.5343080229485738, + "kl": 0.0283203125, + "learning_rate": 3.544378698224852e-07, + "loss": 0.0011, + "reward": 0.1484375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1091 + }, + { + "completion_length": 134.1484375, + "epoch": 0.6461538461538462, + "grad_norm": 0.774033809828994, + "kl": 0.034423828125, + "learning_rate": 3.5384615384615386e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1092 + }, + { + "completion_length": 129.328125, + "epoch": 0.6467455621301775, + "grad_norm": 0.5685714294404755, + "kl": 0.028076171875, + "learning_rate": 3.5325443786982244e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1093 + }, + { + "completion_length": 118.96875, + "epoch": 0.6473372781065089, + "grad_norm": 0.6036448764733421, + "kl": 0.03369140625, + "learning_rate": 3.526627218934911e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1094 + }, + { + "completion_length": 129.25, + "epoch": 0.6479289940828402, + "grad_norm": 0.5455830730142456, + "kl": 0.023193359375, + "learning_rate": 3.5207100591715975e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1095 + }, + { + "completion_length": 128.1640625, + "epoch": 0.6485207100591716, + "grad_norm": 0.22767391739329473, + "kl": 0.029052734375, + "learning_rate": 3.514792899408284e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1096 + }, + { + "completion_length": 124.703125, + "epoch": 0.649112426035503, + "grad_norm": 0.6278756340976858, + "kl": 0.040771484375, + "learning_rate": 3.50887573964497e-07, + "loss": 0.0016, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1097 + }, + { + "completion_length": 122.390625, + "epoch": 0.6497041420118344, + "grad_norm": 0.5564346830154557, + "kl": 0.034423828125, + "learning_rate": 3.502958579881657e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1098 + }, + { + "completion_length": 124.3515625, + "epoch": 0.6502958579881657, + "grad_norm": 0.2374094543111302, + "kl": 0.032470703125, + "learning_rate": 3.4970414201183427e-07, + "loss": 0.0013, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1099 + }, + { + "completion_length": 117.6875, + "epoch": 0.650887573964497, + "grad_norm": 0.5900511901837107, + "kl": 0.0400390625, + "learning_rate": 3.4911242603550296e-07, + "loss": 0.0016, + "reward": 0.203125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.203125, + "step": 1100 + }, + { + "completion_length": 138.1171875, + "epoch": 0.6514792899408284, + "grad_norm": 0.4793700332374219, + "kl": 0.0240478515625, + "learning_rate": 3.485207100591716e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1101 + }, + { + "completion_length": 114.5546875, + "epoch": 0.6520710059171597, + "grad_norm": 0.31166684636747066, + "kl": 0.0341796875, + "learning_rate": 3.479289940828402e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1102 + }, + { + "completion_length": 125.015625, + "epoch": 0.6526627218934912, + "grad_norm": 0.4311440975012103, + "kl": 0.0257568359375, + "learning_rate": 3.4733727810650885e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1103 + }, + { + "completion_length": 138.3828125, + "epoch": 0.6532544378698225, + "grad_norm": 0.7126241558870162, + "kl": 0.0400390625, + "learning_rate": 3.467455621301775e-07, + "loss": 0.0016, + "reward": 0.125, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1104 + }, + { + "completion_length": 126.546875, + "epoch": 0.6538461538461539, + "grad_norm": 0.459602966313143, + "kl": 0.02734375, + "learning_rate": 3.461538461538461e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1105 + }, + { + "completion_length": 121.828125, + "epoch": 0.6544378698224852, + "grad_norm": 0.4966730229584697, + "kl": 0.0277099609375, + "learning_rate": 3.455621301775148e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1106 + }, + { + "completion_length": 126.1875, + "epoch": 0.6550295857988165, + "grad_norm": 0.6862809954567668, + "kl": 0.027099609375, + "learning_rate": 3.4497041420118337e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1107 + }, + { + "completion_length": 127.578125, + "epoch": 0.6556213017751479, + "grad_norm": 0.5967962976845832, + "kl": 0.0272216796875, + "learning_rate": 3.4437869822485205e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1108 + }, + { + "completion_length": 127.2421875, + "epoch": 0.6562130177514793, + "grad_norm": 0.3274509221347917, + "kl": 0.021484375, + "learning_rate": 3.4378698224852073e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1109 + }, + { + "completion_length": 131.25, + "epoch": 0.6568047337278107, + "grad_norm": 1.1204700372272671, + "kl": 0.0361328125, + "learning_rate": 3.431952662721893e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1110 + }, + { + "completion_length": 137.125, + "epoch": 0.657396449704142, + "grad_norm": 0.5073005999420922, + "kl": 0.0341796875, + "learning_rate": 3.42603550295858e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1111 + }, + { + "completion_length": 133.75, + "epoch": 0.6579881656804734, + "grad_norm": 0.4767554614205825, + "kl": 0.0302734375, + "learning_rate": 3.420118343195266e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1112 + }, + { + "completion_length": 120.59375, + "epoch": 0.6585798816568047, + "grad_norm": 0.5825690087658424, + "kl": 0.0302734375, + "learning_rate": 3.4142011834319525e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1113 + }, + { + "completion_length": 125.953125, + "epoch": 0.659171597633136, + "grad_norm": 1.1306129872636148, + "kl": 0.028076171875, + "learning_rate": 3.408284023668639e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1114 + }, + { + "completion_length": 133.9296875, + "epoch": 0.6597633136094675, + "grad_norm": 0.5902715605195116, + "kl": 0.0201416015625, + "learning_rate": 3.4023668639053256e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1115 + }, + { + "completion_length": 125.703125, + "epoch": 0.6603550295857988, + "grad_norm": 0.6679157295695715, + "kl": 0.028564453125, + "learning_rate": 3.3964497041420114e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1116 + }, + { + "completion_length": 131.140625, + "epoch": 0.6609467455621302, + "grad_norm": 0.4449417518736341, + "kl": 0.0235595703125, + "learning_rate": 3.390532544378698e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1117 + }, + { + "completion_length": 135.6796875, + "epoch": 0.6615384615384615, + "grad_norm": 0.42070544244113073, + "kl": 0.0284423828125, + "learning_rate": 3.3846153846153845e-07, + "loss": 0.0011, + "reward": 0.1484375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1118 + }, + { + "completion_length": 118.546875, + "epoch": 0.6621301775147929, + "grad_norm": 0.3202975208058883, + "kl": 0.032958984375, + "learning_rate": 3.378698224852071e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1119 + }, + { + "completion_length": 132.625, + "epoch": 0.6627218934911243, + "grad_norm": 0.6652982860407446, + "kl": 0.033203125, + "learning_rate": 3.372781065088757e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1120 + }, + { + "completion_length": 118.390625, + "epoch": 0.6633136094674557, + "grad_norm": 0.6329359449117254, + "kl": 0.03173828125, + "learning_rate": 3.366863905325444e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1121 + }, + { + "completion_length": 124.6328125, + "epoch": 0.663905325443787, + "grad_norm": 0.49358519501989534, + "kl": 0.0341796875, + "learning_rate": 3.36094674556213e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1122 + }, + { + "completion_length": 127.5078125, + "epoch": 0.6644970414201183, + "grad_norm": 0.6269041215061163, + "kl": 0.034423828125, + "learning_rate": 3.3550295857988166e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1123 + }, + { + "completion_length": 125.7890625, + "epoch": 0.6650887573964497, + "grad_norm": 0.4692445996611205, + "kl": 0.037109375, + "learning_rate": 3.349112426035503e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1124 + }, + { + "completion_length": 122.1640625, + "epoch": 0.665680473372781, + "grad_norm": 0.700089729311182, + "kl": 0.035400390625, + "learning_rate": 3.343195266272189e-07, + "loss": 0.0014, + "reward": 0.1484375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1125 + }, + { + "completion_length": 121.4453125, + "epoch": 0.6662721893491125, + "grad_norm": 0.29314637414005673, + "kl": 0.0240478515625, + "learning_rate": 3.3372781065088755e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1126 + }, + { + "completion_length": 114.5546875, + "epoch": 0.6668639053254438, + "grad_norm": 0.47871843486334126, + "kl": 0.0283203125, + "learning_rate": 3.3313609467455623e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1127 + }, + { + "completion_length": 122.5, + "epoch": 0.6674556213017752, + "grad_norm": 0.6437227996477382, + "kl": 0.032958984375, + "learning_rate": 3.325443786982248e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1128 + }, + { + "completion_length": 112.1484375, + "epoch": 0.6680473372781065, + "grad_norm": 0.6113234391701985, + "kl": 0.0274658203125, + "learning_rate": 3.319526627218935e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1129 + }, + { + "completion_length": 128.1171875, + "epoch": 0.6686390532544378, + "grad_norm": 0.6272351733206377, + "kl": 0.0283203125, + "learning_rate": 3.313609467455621e-07, + "loss": 0.0011, + "reward": 0.171875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1130 + }, + { + "completion_length": 116.4921875, + "epoch": 0.6692307692307692, + "grad_norm": 0.07344984344814535, + "kl": 0.0242919921875, + "learning_rate": 3.3076923076923075e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1131 + }, + { + "completion_length": 126.5234375, + "epoch": 0.6698224852071006, + "grad_norm": 0.5079074578098501, + "kl": 0.023681640625, + "learning_rate": 3.301775147928994e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1132 + }, + { + "completion_length": 115.9453125, + "epoch": 0.670414201183432, + "grad_norm": 0.4484468352194987, + "kl": 0.0255126953125, + "learning_rate": 3.29585798816568e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1133 + }, + { + "completion_length": 118.7890625, + "epoch": 0.6710059171597633, + "grad_norm": 0.3659503879551426, + "kl": 0.03125, + "learning_rate": 3.2899408284023664e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1134 + }, + { + "completion_length": 114.0546875, + "epoch": 0.6715976331360947, + "grad_norm": 0.696353275471397, + "kl": 0.0286865234375, + "learning_rate": 3.284023668639053e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1135 + }, + { + "completion_length": 119.2578125, + "epoch": 0.672189349112426, + "grad_norm": 3.321619088630428, + "kl": 0.028564453125, + "learning_rate": 3.278106508875739e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1136 + }, + { + "completion_length": 118.515625, + "epoch": 0.6727810650887573, + "grad_norm": 0.8744953537230288, + "kl": 0.0302734375, + "learning_rate": 3.272189349112426e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1137 + }, + { + "completion_length": 117.546875, + "epoch": 0.6733727810650888, + "grad_norm": 0.6733685002236147, + "kl": 0.037109375, + "learning_rate": 3.2662721893491127e-07, + "loss": 0.0015, + "reward": 0.1484375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1138 + }, + { + "completion_length": 121.953125, + "epoch": 0.6739644970414201, + "grad_norm": 0.4041494576712075, + "kl": 0.033447265625, + "learning_rate": 3.2603550295857984e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1139 + }, + { + "completion_length": 110.3125, + "epoch": 0.6745562130177515, + "grad_norm": 0.6227021070287332, + "kl": 0.032958984375, + "learning_rate": 3.254437869822485e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1140 + }, + { + "completion_length": 116.609375, + "epoch": 0.6751479289940828, + "grad_norm": 0.33034606662956345, + "kl": 0.0291748046875, + "learning_rate": 3.2485207100591716e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1141 + }, + { + "completion_length": 117.78125, + "epoch": 0.6757396449704142, + "grad_norm": 0.7076550707611522, + "kl": 0.04150390625, + "learning_rate": 3.242603550295858e-07, + "loss": 0.0017, + "reward": 0.2109375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2109375, + "step": 1142 + }, + { + "completion_length": 119.328125, + "epoch": 0.6763313609467455, + "grad_norm": 0.46652852185210186, + "kl": 0.03662109375, + "learning_rate": 3.236686390532544e-07, + "loss": 0.0015, + "reward": 0.1640625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1143 + }, + { + "completion_length": 120.9296875, + "epoch": 0.676923076923077, + "grad_norm": 0.3333818044596869, + "kl": 0.03271484375, + "learning_rate": 3.230769230769231e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1144 + }, + { + "completion_length": 114.6953125, + "epoch": 0.6775147928994083, + "grad_norm": 0.6179792999599888, + "kl": 0.0213623046875, + "learning_rate": 3.224852071005917e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1145 + }, + { + "completion_length": 117.2421875, + "epoch": 0.6781065088757396, + "grad_norm": 0.27756794235639076, + "kl": 0.0208740234375, + "learning_rate": 3.2189349112426036e-07, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1146 + }, + { + "completion_length": 116.359375, + "epoch": 0.678698224852071, + "grad_norm": 0.4927135077315978, + "kl": 0.03466796875, + "learning_rate": 3.21301775147929e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1147 + }, + { + "completion_length": 128.5, + "epoch": 0.6792899408284023, + "grad_norm": 0.4546352386150564, + "kl": 0.0228271484375, + "learning_rate": 3.207100591715976e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1148 + }, + { + "completion_length": 122.734375, + "epoch": 0.6798816568047338, + "grad_norm": 0.7633293461095161, + "kl": 0.03271484375, + "learning_rate": 3.2011834319526625e-07, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1149 + }, + { + "completion_length": 115.984375, + "epoch": 0.6804733727810651, + "grad_norm": 0.6249443947487604, + "kl": 0.039794921875, + "learning_rate": 3.1952662721893493e-07, + "loss": 0.0016, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1150 + }, + { + "completion_length": 114.4375, + "epoch": 0.6810650887573965, + "grad_norm": 0.5678644992546122, + "kl": 0.03271484375, + "learning_rate": 3.189349112426035e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1151 + }, + { + "completion_length": 125.359375, + "epoch": 0.6816568047337278, + "grad_norm": 0.4442404955696638, + "kl": 0.0257568359375, + "learning_rate": 3.183431952662722e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1152 + }, + { + "completion_length": 117.78125, + "epoch": 0.6822485207100591, + "grad_norm": 0.32195783461757643, + "kl": 0.033935546875, + "learning_rate": 3.177514792899408e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1153 + }, + { + "completion_length": 119.359375, + "epoch": 0.6828402366863905, + "grad_norm": 0.5904787371745609, + "kl": 0.025390625, + "learning_rate": 3.1715976331360945e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1154 + }, + { + "completion_length": 117.0546875, + "epoch": 0.6834319526627219, + "grad_norm": 0.28597436016831357, + "kl": 0.032470703125, + "learning_rate": 3.165680473372781e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1155 + }, + { + "completion_length": 118.5078125, + "epoch": 0.6840236686390533, + "grad_norm": 0.5719688423690324, + "kl": 0.042236328125, + "learning_rate": 3.1597633136094676e-07, + "loss": 0.0017, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1156 + }, + { + "completion_length": 126.3984375, + "epoch": 0.6846153846153846, + "grad_norm": 0.5740686859532191, + "kl": 0.0301513671875, + "learning_rate": 3.1538461538461534e-07, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1157 + }, + { + "completion_length": 116.921875, + "epoch": 0.685207100591716, + "grad_norm": 0.4977340134238347, + "kl": 0.0272216796875, + "learning_rate": 3.14792899408284e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1158 + }, + { + "completion_length": 114.671875, + "epoch": 0.6857988165680473, + "grad_norm": 0.40746065596338166, + "kl": 0.0308837890625, + "learning_rate": 3.142011834319526e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1159 + }, + { + "completion_length": 117.984375, + "epoch": 0.6863905325443787, + "grad_norm": 0.9635501623712249, + "kl": 0.041015625, + "learning_rate": 3.136094674556213e-07, + "loss": 0.0016, + "reward": 0.09375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1160 + }, + { + "completion_length": 115.9453125, + "epoch": 0.6869822485207101, + "grad_norm": 0.6300753732491076, + "kl": 0.048095703125, + "learning_rate": 3.130177514792899e-07, + "loss": 0.0019, + "reward": 0.25, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.25, + "step": 1161 + }, + { + "completion_length": 113.4140625, + "epoch": 0.6875739644970414, + "grad_norm": 0.6464981852488283, + "kl": 0.034912109375, + "learning_rate": 3.1242603550295854e-07, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1162 + }, + { + "completion_length": 121.65625, + "epoch": 0.6881656804733728, + "grad_norm": 0.30056160668009463, + "kl": 0.023193359375, + "learning_rate": 3.118343195266272e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1163 + }, + { + "completion_length": 112.9140625, + "epoch": 0.6887573964497041, + "grad_norm": 1.1150116974679234, + "kl": 0.028564453125, + "learning_rate": 3.1124260355029586e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.11179219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1164 + }, + { + "completion_length": 112.1875, + "epoch": 0.6893491124260355, + "grad_norm": 0.6265044746513825, + "kl": 0.03125, + "learning_rate": 3.1065088757396443e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1165 + }, + { + "completion_length": 115.7265625, + "epoch": 0.6899408284023668, + "grad_norm": 0.6750879794886212, + "kl": 0.0274658203125, + "learning_rate": 3.100591715976331e-07, + "loss": 0.0011, + "reward": 0.171875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1166 + }, + { + "completion_length": 127.09375, + "epoch": 0.6905325443786983, + "grad_norm": 0.7506266446103717, + "kl": 0.0260009765625, + "learning_rate": 3.094674556213018e-07, + "loss": 0.001, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1167 + }, + { + "completion_length": 124.3671875, + "epoch": 0.6911242603550296, + "grad_norm": 1.3191316666382737, + "kl": 0.04052734375, + "learning_rate": 3.088757396449704e-07, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1168 + }, + { + "completion_length": 112.203125, + "epoch": 0.691715976331361, + "grad_norm": 0.31274910860926686, + "kl": 0.03125, + "learning_rate": 3.0828402366863906e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1169 + }, + { + "completion_length": 116.609375, + "epoch": 0.6923076923076923, + "grad_norm": 1.1168803764909292, + "kl": 0.03271484375, + "learning_rate": 3.076923076923077e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1170 + }, + { + "completion_length": 123.0703125, + "epoch": 0.6928994082840236, + "grad_norm": 0.14681668621455604, + "kl": 0.0250244140625, + "learning_rate": 3.071005917159763e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1171 + }, + { + "completion_length": 116.515625, + "epoch": 0.693491124260355, + "grad_norm": 0.2998540210436418, + "kl": 0.033447265625, + "learning_rate": 3.0650887573964495e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1172 + }, + { + "completion_length": 122.421875, + "epoch": 0.6940828402366864, + "grad_norm": 0.5234828243607553, + "kl": 0.0281982421875, + "learning_rate": 3.0591715976331363e-07, + "loss": 0.0011, + "reward": 0.1484375, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1173 + }, + { + "completion_length": 116.34375, + "epoch": 0.6946745562130178, + "grad_norm": 0.5387959045471048, + "kl": 0.04296875, + "learning_rate": 3.053254437869822e-07, + "loss": 0.0017, + "reward": 0.1484375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1174 + }, + { + "completion_length": 119.40625, + "epoch": 0.6952662721893491, + "grad_norm": 0.24520123171412123, + "kl": 0.0213623046875, + "learning_rate": 3.047337278106509e-07, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1175 + }, + { + "completion_length": 121.5703125, + "epoch": 0.6958579881656805, + "grad_norm": 0.4774964020050323, + "kl": 0.027587890625, + "learning_rate": 3.041420118343195e-07, + "loss": 0.0011, + "reward": 0.1640625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1176 + }, + { + "completion_length": 116.515625, + "epoch": 0.6964497041420118, + "grad_norm": 2.052248282967972, + "kl": 0.030517578125, + "learning_rate": 3.0355029585798815e-07, + "loss": 0.0012, + "reward": 0.15625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1177 + }, + { + "completion_length": 119.671875, + "epoch": 0.6970414201183432, + "grad_norm": 0.7262886514134909, + "kl": 0.03271484375, + "learning_rate": 3.029585798816568e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1178 + }, + { + "completion_length": 126.4375, + "epoch": 0.6976331360946746, + "grad_norm": 0.6209012262091416, + "kl": 0.0283203125, + "learning_rate": 3.0236686390532547e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1179 + }, + { + "completion_length": 119.0546875, + "epoch": 0.6982248520710059, + "grad_norm": 0.4853821589795314, + "kl": 0.022705078125, + "learning_rate": 3.0177514792899404e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1180 + }, + { + "completion_length": 116.015625, + "epoch": 0.6988165680473373, + "grad_norm": 0.44681074810693727, + "kl": 0.0244140625, + "learning_rate": 3.011834319526627e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1181 + }, + { + "completion_length": 120.6328125, + "epoch": 0.6994082840236686, + "grad_norm": 0.5790846833537607, + "kl": 0.017822265625, + "learning_rate": 3.0059171597633136e-07, + "loss": 0.0007, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1182 + }, + { + "completion_length": 122.2109375, + "epoch": 0.7, + "grad_norm": 0.5282834164178656, + "kl": 0.0235595703125, + "learning_rate": 3e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1183 + }, + { + "completion_length": 121.2109375, + "epoch": 0.7005917159763314, + "grad_norm": 0.5292897490587224, + "kl": 0.0279541015625, + "learning_rate": 2.994082840236686e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1184 + }, + { + "completion_length": 124.4765625, + "epoch": 0.7011834319526628, + "grad_norm": 0.8754844798304032, + "kl": 0.0341796875, + "learning_rate": 2.988165680473373e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1185 + }, + { + "completion_length": 120.3046875, + "epoch": 0.7017751479289941, + "grad_norm": 0.4156204466477295, + "kl": 0.02783203125, + "learning_rate": 2.982248520710059e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1186 + }, + { + "completion_length": 130.1953125, + "epoch": 0.7023668639053254, + "grad_norm": 0.12161173513325987, + "kl": 0.0216064453125, + "learning_rate": 2.9763313609467456e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1187 + }, + { + "completion_length": 128.65625, + "epoch": 0.7029585798816568, + "grad_norm": 0.6026645575716368, + "kl": 0.02490234375, + "learning_rate": 2.9704142011834314e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1188 + }, + { + "completion_length": 117.5078125, + "epoch": 0.7035502958579881, + "grad_norm": 0.5834806559837289, + "kl": 0.0263671875, + "learning_rate": 2.964497041420118e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1189 + }, + { + "completion_length": 123.8515625, + "epoch": 0.7041420118343196, + "grad_norm": 0.5016483060942221, + "kl": 0.029541015625, + "learning_rate": 2.9585798816568045e-07, + "loss": 0.0012, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1190 + }, + { + "completion_length": 119.109375, + "epoch": 0.7047337278106509, + "grad_norm": 0.7542952475585215, + "kl": 0.034912109375, + "learning_rate": 2.952662721893491e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1191 + }, + { + "completion_length": 117.6953125, + "epoch": 0.7053254437869823, + "grad_norm": 0.5459852416336076, + "kl": 0.029541015625, + "learning_rate": 2.946745562130177e-07, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1192 + }, + { + "completion_length": 116.6796875, + "epoch": 0.7059171597633136, + "grad_norm": 0.4587022533215746, + "kl": 0.029052734375, + "learning_rate": 2.940828402366864e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1193 + }, + { + "completion_length": 129.140625, + "epoch": 0.7065088757396449, + "grad_norm": 0.5175887233299398, + "kl": 0.031005859375, + "learning_rate": 2.9349112426035497e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1194 + }, + { + "completion_length": 117.96875, + "epoch": 0.7071005917159763, + "grad_norm": 0.7893301862243752, + "kl": 0.04052734375, + "learning_rate": 2.9289940828402365e-07, + "loss": 0.0016, + "reward": 0.234375, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.234375, + "step": 1195 + }, + { + "completion_length": 120.765625, + "epoch": 0.7076923076923077, + "grad_norm": 0.49551560894477764, + "kl": 0.0262451171875, + "learning_rate": 2.9230769230769234e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1196 + }, + { + "completion_length": 119.3203125, + "epoch": 0.7082840236686391, + "grad_norm": 0.4161391251329746, + "kl": 0.0230712890625, + "learning_rate": 2.917159763313609e-07, + "loss": 0.0009, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1197 + }, + { + "completion_length": 121.9765625, + "epoch": 0.7088757396449704, + "grad_norm": 0.2925782509776678, + "kl": 0.0291748046875, + "learning_rate": 2.911242603550296e-07, + "loss": 0.0012, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1198 + }, + { + "completion_length": 124.46875, + "epoch": 0.7094674556213018, + "grad_norm": 0.29160321361104663, + "kl": 0.0230712890625, + "learning_rate": 2.905325443786982e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1199 + }, + { + "completion_length": 128.4375, + "epoch": 0.7100591715976331, + "grad_norm": 0.6038198368877594, + "kl": 0.0286865234375, + "learning_rate": 2.8994082840236686e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1200 + }, + { + "completion_length": 119.7265625, + "epoch": 0.7106508875739646, + "grad_norm": 0.2614516278644708, + "kl": 0.01904296875, + "learning_rate": 2.893491124260355e-07, + "loss": 0.0008, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1201 + }, + { + "completion_length": 120.234375, + "epoch": 0.7112426035502959, + "grad_norm": 0.6449447974290843, + "kl": 0.033935546875, + "learning_rate": 2.8875739644970417e-07, + "loss": 0.0014, + "reward": 0.1640625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1202 + }, + { + "completion_length": 114.171875, + "epoch": 0.7118343195266272, + "grad_norm": 0.06999434971982176, + "kl": 0.0205078125, + "learning_rate": 2.8816568047337275e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1203 + }, + { + "completion_length": 123.828125, + "epoch": 0.7124260355029586, + "grad_norm": 0.4370722344791253, + "kl": 0.029052734375, + "learning_rate": 2.8757396449704143e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1204 + }, + { + "completion_length": 122.546875, + "epoch": 0.7130177514792899, + "grad_norm": 0.4845103218480956, + "kl": 0.0341796875, + "learning_rate": 2.8698224852071006e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1205 + }, + { + "completion_length": 123.0703125, + "epoch": 0.7136094674556213, + "grad_norm": 1.4310283312051457, + "kl": 0.036865234375, + "learning_rate": 2.863905325443787e-07, + "loss": 0.0015, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1206 + }, + { + "completion_length": 136.28125, + "epoch": 0.7142011834319527, + "grad_norm": 0.7813740331707243, + "kl": 0.0283203125, + "learning_rate": 2.857988165680473e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.10583597421646118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1207 + }, + { + "completion_length": 131.8984375, + "epoch": 0.714792899408284, + "grad_norm": 0.507246583021854, + "kl": 0.022705078125, + "learning_rate": 2.85207100591716e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1208 + }, + { + "completion_length": 143.8515625, + "epoch": 0.7153846153846154, + "grad_norm": 0.2590210616934181, + "kl": 0.027587890625, + "learning_rate": 2.846153846153846e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1209 + }, + { + "completion_length": 130.25, + "epoch": 0.7159763313609467, + "grad_norm": 0.4125845082768469, + "kl": 0.03271484375, + "learning_rate": 2.8402366863905326e-07, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1210 + }, + { + "completion_length": 123.125, + "epoch": 0.7165680473372781, + "grad_norm": 0.588617483397953, + "kl": 0.034912109375, + "learning_rate": 2.834319526627219e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1211 + }, + { + "completion_length": 127.0078125, + "epoch": 0.7171597633136094, + "grad_norm": 0.4279777891152955, + "kl": 0.02587890625, + "learning_rate": 2.828402366863905e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1212 + }, + { + "completion_length": 126.5625, + "epoch": 0.7177514792899409, + "grad_norm": 0.3561671964726857, + "kl": 0.02197265625, + "learning_rate": 2.8224852071005915e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1213 + }, + { + "completion_length": 129.453125, + "epoch": 0.7183431952662722, + "grad_norm": 0.5915283120454404, + "kl": 0.0263671875, + "learning_rate": 2.816568047337278e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1214 + }, + { + "completion_length": 130.765625, + "epoch": 0.7189349112426036, + "grad_norm": 0.06331823330157894, + "kl": 0.0206298828125, + "learning_rate": 2.810650887573964e-07, + "loss": 0.0008, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1215 + }, + { + "completion_length": 140.125, + "epoch": 0.7195266272189349, + "grad_norm": 0.6091921891278745, + "kl": 0.041259765625, + "learning_rate": 2.804733727810651e-07, + "loss": 0.0016, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1216 + }, + { + "completion_length": 132.5078125, + "epoch": 0.7201183431952662, + "grad_norm": 0.2700545185981257, + "kl": 0.0264892578125, + "learning_rate": 2.7988165680473367e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1217 + }, + { + "completion_length": 140.8984375, + "epoch": 0.7207100591715976, + "grad_norm": 0.46833108543801916, + "kl": 0.0225830078125, + "learning_rate": 2.7928994082840235e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1218 + }, + { + "completion_length": 128.6171875, + "epoch": 0.721301775147929, + "grad_norm": 0.5749003730152508, + "kl": 0.037353515625, + "learning_rate": 2.78698224852071e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1219 + }, + { + "completion_length": 142.0625, + "epoch": 0.7218934911242604, + "grad_norm": 0.5188132559820664, + "kl": 0.0267333984375, + "learning_rate": 2.781065088757396e-07, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1220 + }, + { + "completion_length": 135.40625, + "epoch": 0.7224852071005917, + "grad_norm": 0.46176098673319127, + "kl": 0.018310546875, + "learning_rate": 2.7751479289940824e-07, + "loss": 0.0007, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1221 + }, + { + "completion_length": 138.390625, + "epoch": 0.7230769230769231, + "grad_norm": 0.38609090994644374, + "kl": 0.0230712890625, + "learning_rate": 2.7692307692307693e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1222 + }, + { + "completion_length": 128.75, + "epoch": 0.7236686390532544, + "grad_norm": 0.4074487703618128, + "kl": 0.025146484375, + "learning_rate": 2.763313609467455e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1223 + }, + { + "completion_length": 139.3125, + "epoch": 0.7242603550295857, + "grad_norm": 0.5368194690321352, + "kl": 0.0311279296875, + "learning_rate": 2.757396449704142e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1224 + }, + { + "completion_length": 130.8515625, + "epoch": 0.7248520710059172, + "grad_norm": 0.4383832225586893, + "kl": 0.024658203125, + "learning_rate": 2.7514792899408287e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1225 + }, + { + "completion_length": 127.703125, + "epoch": 0.7254437869822485, + "grad_norm": 0.832465916978995, + "kl": 0.023681640625, + "learning_rate": 2.7455621301775145e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1226 + }, + { + "completion_length": 131.4453125, + "epoch": 0.7260355029585799, + "grad_norm": 0.5038438677574101, + "kl": 0.028564453125, + "learning_rate": 2.7396449704142013e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1227 + }, + { + "completion_length": 139.03125, + "epoch": 0.7266272189349112, + "grad_norm": 0.40148759389814787, + "kl": 0.0208740234375, + "learning_rate": 2.7337278106508876e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1228 + }, + { + "completion_length": 132.2734375, + "epoch": 0.7272189349112426, + "grad_norm": 0.36553005539639416, + "kl": 0.0322265625, + "learning_rate": 2.727810650887574e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1229 + }, + { + "completion_length": 139.9375, + "epoch": 0.727810650887574, + "grad_norm": 0.39690746373928504, + "kl": 0.0238037109375, + "learning_rate": 2.72189349112426e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1230 + }, + { + "completion_length": 122.8046875, + "epoch": 0.7284023668639054, + "grad_norm": 0.10351125783648323, + "kl": 0.0303955078125, + "learning_rate": 2.715976331360947e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1231 + }, + { + "completion_length": 137.5390625, + "epoch": 0.7289940828402367, + "grad_norm": 0.44176224625971183, + "kl": 0.0220947265625, + "learning_rate": 2.710059171597633e-07, + "loss": 0.0009, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1232 + }, + { + "completion_length": 126.875, + "epoch": 0.729585798816568, + "grad_norm": 0.6080079993954344, + "kl": 0.0267333984375, + "learning_rate": 2.7041420118343196e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1233 + }, + { + "completion_length": 135.765625, + "epoch": 0.7301775147928994, + "grad_norm": 0.7806520162240217, + "kl": 0.031494140625, + "learning_rate": 2.698224852071006e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1234 + }, + { + "completion_length": 130.859375, + "epoch": 0.7307692307692307, + "grad_norm": 0.4920372356075306, + "kl": 0.0218505859375, + "learning_rate": 2.692307692307692e-07, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1235 + }, + { + "completion_length": 134.1171875, + "epoch": 0.7313609467455622, + "grad_norm": 0.2507905120360076, + "kl": 0.0184326171875, + "learning_rate": 2.6863905325443785e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1236 + }, + { + "completion_length": 121.859375, + "epoch": 0.7319526627218935, + "grad_norm": 0.48633632102689983, + "kl": 0.0245361328125, + "learning_rate": 2.6804733727810654e-07, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1237 + }, + { + "completion_length": 130.09375, + "epoch": 0.7325443786982249, + "grad_norm": 0.4202270416270141, + "kl": 0.0255126953125, + "learning_rate": 2.674556213017751e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1238 + }, + { + "completion_length": 124.5078125, + "epoch": 0.7331360946745562, + "grad_norm": 0.6753119667993306, + "kl": 0.035400390625, + "learning_rate": 2.668639053254438e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1239 + }, + { + "completion_length": 129.3984375, + "epoch": 0.7337278106508875, + "grad_norm": 0.2840232595838978, + "kl": 0.029296875, + "learning_rate": 2.662721893491124e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1240 + }, + { + "completion_length": 128.2890625, + "epoch": 0.7343195266272189, + "grad_norm": 0.36328527568270774, + "kl": 0.0186767578125, + "learning_rate": 2.6568047337278106e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1241 + }, + { + "completion_length": 132.09375, + "epoch": 0.7349112426035503, + "grad_norm": 0.11416957879861332, + "kl": 0.02392578125, + "learning_rate": 2.650887573964497e-07, + "loss": 0.001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1242 + }, + { + "completion_length": 128.328125, + "epoch": 0.7355029585798817, + "grad_norm": 0.2721446158661043, + "kl": 0.023681640625, + "learning_rate": 2.644970414201183e-07, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1243 + }, + { + "completion_length": 128.9140625, + "epoch": 0.736094674556213, + "grad_norm": 0.4885983856783718, + "kl": 0.031494140625, + "learning_rate": 2.6390532544378695e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1244 + }, + { + "completion_length": 125.7265625, + "epoch": 0.7366863905325444, + "grad_norm": 0.7614484874383499, + "kl": 0.033203125, + "learning_rate": 2.6331360946745563e-07, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1245 + }, + { + "completion_length": 123.0390625, + "epoch": 0.7372781065088757, + "grad_norm": 0.30823106832582053, + "kl": 0.025390625, + "learning_rate": 2.627218934911242e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1246 + }, + { + "completion_length": 124.015625, + "epoch": 0.737869822485207, + "grad_norm": 0.6137757174844274, + "kl": 0.027587890625, + "learning_rate": 2.621301775147929e-07, + "loss": 0.0011, + "reward": 0.1328125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1247 + }, + { + "completion_length": 130.140625, + "epoch": 0.7384615384615385, + "grad_norm": 0.26960490775552004, + "kl": 0.0228271484375, + "learning_rate": 2.615384615384615e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1248 + }, + { + "completion_length": 133.4296875, + "epoch": 0.7390532544378698, + "grad_norm": 0.4373165305707865, + "kl": 0.02978515625, + "learning_rate": 2.6094674556213015e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1249 + }, + { + "completion_length": 121.4609375, + "epoch": 0.7396449704142012, + "grad_norm": 0.44532974330909236, + "kl": 0.0185546875, + "learning_rate": 2.603550295857988e-07, + "loss": 0.0007, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1250 + }, + { + "completion_length": 120.0078125, + "epoch": 0.7402366863905325, + "grad_norm": 0.4178260383105502, + "kl": 0.0252685546875, + "learning_rate": 2.5976331360946746e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1251 + }, + { + "completion_length": 132.75, + "epoch": 0.7408284023668639, + "grad_norm": 0.4193821559812818, + "kl": 0.02587890625, + "learning_rate": 2.5917159763313604e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1252 + }, + { + "completion_length": 121.5, + "epoch": 0.7414201183431952, + "grad_norm": 0.549516265449472, + "kl": 0.0263671875, + "learning_rate": 2.585798816568047e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1253 + }, + { + "completion_length": 121.3828125, + "epoch": 0.7420118343195267, + "grad_norm": 0.7961923702253572, + "kl": 0.028076171875, + "learning_rate": 2.579881656804734e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1254 + }, + { + "completion_length": 131.0234375, + "epoch": 0.742603550295858, + "grad_norm": 0.7517470659834267, + "kl": 0.04052734375, + "learning_rate": 2.57396449704142e-07, + "loss": 0.0016, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1255 + }, + { + "completion_length": 126.296875, + "epoch": 0.7431952662721893, + "grad_norm": 0.41788213775090693, + "kl": 0.0283203125, + "learning_rate": 2.5680473372781066e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1256 + }, + { + "completion_length": 130.296875, + "epoch": 0.7437869822485207, + "grad_norm": 0.5239853922039225, + "kl": 0.02734375, + "learning_rate": 2.562130177514793e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1257 + }, + { + "completion_length": 125.8359375, + "epoch": 0.744378698224852, + "grad_norm": 0.48416633645088647, + "kl": 0.0252685546875, + "learning_rate": 2.556213017751479e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1258 + }, + { + "completion_length": 120.0703125, + "epoch": 0.7449704142011835, + "grad_norm": 0.25967011773176024, + "kl": 0.0230712890625, + "learning_rate": 2.5502958579881655e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1259 + }, + { + "completion_length": 125.8203125, + "epoch": 0.7455621301775148, + "grad_norm": 0.2701889812177114, + "kl": 0.023681640625, + "learning_rate": 2.5443786982248524e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1260 + }, + { + "completion_length": 117.5703125, + "epoch": 0.7461538461538462, + "grad_norm": 0.26504009476728924, + "kl": 0.0242919921875, + "learning_rate": 2.538461538461538e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1261 + }, + { + "completion_length": 123.0546875, + "epoch": 0.7467455621301775, + "grad_norm": 0.5995160174903087, + "kl": 0.0224609375, + "learning_rate": 2.532544378698225e-07, + "loss": 0.0009, + "reward": 0.171875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1262 + }, + { + "completion_length": 118.015625, + "epoch": 0.7473372781065089, + "grad_norm": 0.7185372544448768, + "kl": 0.0341796875, + "learning_rate": 2.5266272189349113e-07, + "loss": 0.0014, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1263 + }, + { + "completion_length": 126.0703125, + "epoch": 0.7479289940828402, + "grad_norm": 0.44127937054501953, + "kl": 0.018310546875, + "learning_rate": 2.5207100591715976e-07, + "loss": 0.0007, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1264 + }, + { + "completion_length": 124.1875, + "epoch": 0.7485207100591716, + "grad_norm": 0.46689305077941906, + "kl": 0.02099609375, + "learning_rate": 2.514792899408284e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1265 + }, + { + "completion_length": 123.6328125, + "epoch": 0.749112426035503, + "grad_norm": 0.5775796112541828, + "kl": 0.03515625, + "learning_rate": 2.5088757396449707e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1266 + }, + { + "completion_length": 114.84375, + "epoch": 0.7497041420118343, + "grad_norm": 0.749999869974785, + "kl": 0.0291748046875, + "learning_rate": 2.5029585798816565e-07, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1267 + }, + { + "completion_length": 119.9375, + "epoch": 0.7502958579881657, + "grad_norm": 0.3010839646885094, + "kl": 0.03466796875, + "learning_rate": 2.4970414201183433e-07, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1268 + }, + { + "completion_length": 126.0546875, + "epoch": 0.750887573964497, + "grad_norm": 0.8684138096960948, + "kl": 0.04736328125, + "learning_rate": 2.4911242603550296e-07, + "loss": 0.0019, + "reward": 0.28125, + "reward_std": 0.10341878235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.28125, + "step": 1269 + }, + { + "completion_length": 132.6171875, + "epoch": 0.7514792899408284, + "grad_norm": 0.06222205690648557, + "kl": 0.024658203125, + "learning_rate": 2.485207100591716e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1270 + }, + { + "completion_length": 113.9453125, + "epoch": 0.7520710059171598, + "grad_norm": 0.7483742510122264, + "kl": 0.030029296875, + "learning_rate": 2.479289940828402e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1271 + }, + { + "completion_length": 114.578125, + "epoch": 0.7526627218934911, + "grad_norm": 0.3361633686992279, + "kl": 0.0291748046875, + "learning_rate": 2.4733727810650885e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1272 + }, + { + "completion_length": 123.8203125, + "epoch": 0.7532544378698225, + "grad_norm": 0.2050289077396947, + "kl": 0.0245361328125, + "learning_rate": 2.467455621301775e-07, + "loss": 0.001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1273 + }, + { + "completion_length": 121.84375, + "epoch": 0.7538461538461538, + "grad_norm": 0.3643021658717396, + "kl": 0.022216796875, + "learning_rate": 2.4615384615384616e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1274 + }, + { + "completion_length": 126.8203125, + "epoch": 0.7544378698224852, + "grad_norm": 0.32516830831242327, + "kl": 0.0255126953125, + "learning_rate": 2.455621301775148e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1275 + }, + { + "completion_length": 119.2421875, + "epoch": 0.7550295857988165, + "grad_norm": 0.6837494157101559, + "kl": 0.0262451171875, + "learning_rate": 2.449704142011834e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1276 + }, + { + "completion_length": 126.8984375, + "epoch": 0.755621301775148, + "grad_norm": 0.40678095854310375, + "kl": 0.0211181640625, + "learning_rate": 2.4437869822485205e-07, + "loss": 0.0008, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.1171875, + "step": 1277 + }, + { + "completion_length": 122.921875, + "epoch": 0.7562130177514793, + "grad_norm": 0.7825654917332846, + "kl": 0.02392578125, + "learning_rate": 2.437869822485207e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.109375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1278 + }, + { + "completion_length": 128.28125, + "epoch": 0.7568047337278107, + "grad_norm": 0.5249757032748422, + "kl": 0.033935546875, + "learning_rate": 2.431952662721893e-07, + "loss": 0.0014, + "reward": 0.1953125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1279 + }, + { + "completion_length": 122.28125, + "epoch": 0.757396449704142, + "grad_norm": 0.47254563904160946, + "kl": 0.026123046875, + "learning_rate": 2.4260355029585794e-07, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1280 + }, + { + "completion_length": 114.90625, + "epoch": 0.7579881656804733, + "grad_norm": 0.3325224946153711, + "kl": 0.01611328125, + "learning_rate": 2.420118343195266e-07, + "loss": 0.0006, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1281 + }, + { + "completion_length": 124.3125, + "epoch": 0.7585798816568048, + "grad_norm": 0.46710960566768767, + "kl": 0.0286865234375, + "learning_rate": 2.4142011834319526e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1282 + }, + { + "completion_length": 114.6796875, + "epoch": 0.7591715976331361, + "grad_norm": 0.4407268458574264, + "kl": 0.0361328125, + "learning_rate": 2.408284023668639e-07, + "loss": 0.0014, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1283 + }, + { + "completion_length": 131.34375, + "epoch": 0.7597633136094675, + "grad_norm": 0.9481148049957511, + "kl": 0.03271484375, + "learning_rate": 2.4023668639053257e-07, + "loss": 0.0013, + "reward": 0.1875, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 1284 + }, + { + "completion_length": 115.7578125, + "epoch": 0.7603550295857988, + "grad_norm": 0.3861653539688578, + "kl": 0.03271484375, + "learning_rate": 2.396449704142012e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1285 + }, + { + "completion_length": 122.1796875, + "epoch": 0.7609467455621302, + "grad_norm": 0.689363735066588, + "kl": 0.034912109375, + "learning_rate": 2.3905325443786983e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1286 + }, + { + "completion_length": 125.2109375, + "epoch": 0.7615384615384615, + "grad_norm": 0.5836423882603741, + "kl": 0.03955078125, + "learning_rate": 2.3846153846153846e-07, + "loss": 0.0016, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1287 + }, + { + "completion_length": 127.8046875, + "epoch": 0.762130177514793, + "grad_norm": 0.4702226581123821, + "kl": 0.0174560546875, + "learning_rate": 2.378698224852071e-07, + "loss": 0.0007, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1288 + }, + { + "completion_length": 123.1171875, + "epoch": 0.7627218934911243, + "grad_norm": 0.5097383412495827, + "kl": 0.029296875, + "learning_rate": 2.3727810650887572e-07, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1289 + }, + { + "completion_length": 116.2109375, + "epoch": 0.7633136094674556, + "grad_norm": 0.4708751436476973, + "kl": 0.0238037109375, + "learning_rate": 2.3668639053254435e-07, + "loss": 0.001, + "reward": 0.140625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1290 + }, + { + "completion_length": 118.671875, + "epoch": 0.763905325443787, + "grad_norm": 0.9392133666288328, + "kl": 0.0279541015625, + "learning_rate": 2.36094674556213e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1291 + }, + { + "completion_length": 119.78125, + "epoch": 0.7644970414201183, + "grad_norm": 0.6351491582246904, + "kl": 0.0260009765625, + "learning_rate": 2.3550295857988164e-07, + "loss": 0.001, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1292 + }, + { + "completion_length": 118.453125, + "epoch": 0.7650887573964497, + "grad_norm": 0.07872126430332436, + "kl": 0.03125, + "learning_rate": 2.3491124260355027e-07, + "loss": 0.0013, + "reward": 0.15625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1293 + }, + { + "completion_length": 116.3125, + "epoch": 0.7656804733727811, + "grad_norm": 0.22764882521802612, + "kl": 0.014892578125, + "learning_rate": 2.3431952662721892e-07, + "loss": 0.0006, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 1294 + }, + { + "completion_length": 126.9140625, + "epoch": 0.7662721893491125, + "grad_norm": 0.8995011296341228, + "kl": 0.027587890625, + "learning_rate": 2.3372781065088755e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1295 + }, + { + "completion_length": 118.7890625, + "epoch": 0.7668639053254438, + "grad_norm": 0.6747365749900935, + "kl": 0.032958984375, + "learning_rate": 2.3313609467455618e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1296 + }, + { + "completion_length": 118.3203125, + "epoch": 0.7674556213017751, + "grad_norm": 0.2778560133135001, + "kl": 0.037353515625, + "learning_rate": 2.3254437869822486e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1297 + }, + { + "completion_length": 113.7890625, + "epoch": 0.7680473372781065, + "grad_norm": 0.30990897860723815, + "kl": 0.0308837890625, + "learning_rate": 2.319526627218935e-07, + "loss": 0.0012, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1298 + }, + { + "completion_length": 124.9140625, + "epoch": 0.7686390532544378, + "grad_norm": 1.012993289069403, + "kl": 0.0400390625, + "learning_rate": 2.3136094674556212e-07, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1299 + }, + { + "completion_length": 126.2578125, + "epoch": 0.7692307692307693, + "grad_norm": 0.7684606873197068, + "kl": 0.03173828125, + "learning_rate": 2.3076923076923078e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1300 + }, + { + "completion_length": 121.546875, + "epoch": 0.7698224852071006, + "grad_norm": 0.5124261798410822, + "kl": 0.0283203125, + "learning_rate": 2.301775147928994e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1301 + }, + { + "completion_length": 131.9140625, + "epoch": 0.770414201183432, + "grad_norm": 0.5530900785252286, + "kl": 0.028564453125, + "learning_rate": 2.2958579881656804e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1302 + }, + { + "completion_length": 127.46875, + "epoch": 0.7710059171597633, + "grad_norm": 0.507130168893199, + "kl": 0.023193359375, + "learning_rate": 2.2899408284023667e-07, + "loss": 0.0009, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1303 + }, + { + "completion_length": 125.265625, + "epoch": 0.7715976331360946, + "grad_norm": 0.4552632292496938, + "kl": 0.032958984375, + "learning_rate": 2.2840236686390533e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1304 + }, + { + "completion_length": 119.1328125, + "epoch": 0.772189349112426, + "grad_norm": 1.1091442815514065, + "kl": 0.036865234375, + "learning_rate": 2.2781065088757396e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1305 + }, + { + "completion_length": 126.203125, + "epoch": 0.7727810650887574, + "grad_norm": 0.26815947635792686, + "kl": 0.019775390625, + "learning_rate": 2.272189349112426e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1306 + }, + { + "completion_length": 124.5625, + "epoch": 0.7733727810650888, + "grad_norm": 0.4207255960874683, + "kl": 0.0224609375, + "learning_rate": 2.2662721893491124e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1307 + }, + { + "completion_length": 121.7890625, + "epoch": 0.7739644970414201, + "grad_norm": 0.7083472132517747, + "kl": 0.0284423828125, + "learning_rate": 2.2603550295857987e-07, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1308 + }, + { + "completion_length": 126.6953125, + "epoch": 0.7745562130177515, + "grad_norm": 0.6746672939792763, + "kl": 0.0284423828125, + "learning_rate": 2.254437869822485e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1309 + }, + { + "completion_length": 122.921875, + "epoch": 0.7751479289940828, + "grad_norm": 0.474946242044051, + "kl": 0.03759765625, + "learning_rate": 2.2485207100591716e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1310 + }, + { + "completion_length": 124.7109375, + "epoch": 0.7757396449704143, + "grad_norm": 1.333718937654965, + "kl": 0.02783203125, + "learning_rate": 2.242603550295858e-07, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1311 + }, + { + "completion_length": 128.3984375, + "epoch": 0.7763313609467456, + "grad_norm": 0.4787682314930239, + "kl": 0.024169921875, + "learning_rate": 2.2366863905325442e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1312 + }, + { + "completion_length": 127.2890625, + "epoch": 0.7769230769230769, + "grad_norm": 0.48734396002181546, + "kl": 0.0311279296875, + "learning_rate": 2.2307692307692308e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1313 + }, + { + "completion_length": 119.5859375, + "epoch": 0.7775147928994083, + "grad_norm": 0.62441875506531, + "kl": 0.03564453125, + "learning_rate": 2.224852071005917e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1314 + }, + { + "completion_length": 125.6875, + "epoch": 0.7781065088757396, + "grad_norm": 0.25743857968387596, + "kl": 0.03125, + "learning_rate": 2.2189349112426034e-07, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1315 + }, + { + "completion_length": 124.75, + "epoch": 0.778698224852071, + "grad_norm": 0.6539871165638096, + "kl": 0.02880859375, + "learning_rate": 2.2130177514792897e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1316 + }, + { + "completion_length": 122.0625, + "epoch": 0.7792899408284024, + "grad_norm": 0.4232106518008989, + "kl": 0.02392578125, + "learning_rate": 2.2071005917159762e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1317 + }, + { + "completion_length": 121.84375, + "epoch": 0.7798816568047338, + "grad_norm": 0.6346233776703973, + "kl": 0.029052734375, + "learning_rate": 2.2011834319526625e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1318 + }, + { + "completion_length": 120.796875, + "epoch": 0.7804733727810651, + "grad_norm": 0.4857029540349173, + "kl": 0.0247802734375, + "learning_rate": 2.1952662721893488e-07, + "loss": 0.001, + "reward": 0.203125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.203125, + "step": 1319 + }, + { + "completion_length": 127.15625, + "epoch": 0.7810650887573964, + "grad_norm": 0.6405438536289504, + "kl": 0.0264892578125, + "learning_rate": 2.1893491124260354e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1320 + }, + { + "completion_length": 117.8359375, + "epoch": 0.7816568047337278, + "grad_norm": 0.5185217907416089, + "kl": 0.0303955078125, + "learning_rate": 2.1834319526627217e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1321 + }, + { + "completion_length": 115.390625, + "epoch": 0.7822485207100591, + "grad_norm": 0.5635184115504249, + "kl": 0.0279541015625, + "learning_rate": 2.177514792899408e-07, + "loss": 0.0011, + "reward": 0.1328125, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1322 + }, + { + "completion_length": 122.9921875, + "epoch": 0.7828402366863906, + "grad_norm": 0.3983695408465268, + "kl": 0.024658203125, + "learning_rate": 2.1715976331360946e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1323 + }, + { + "completion_length": 119.28125, + "epoch": 0.7834319526627219, + "grad_norm": 0.9987099870407833, + "kl": 0.03515625, + "learning_rate": 2.1656804733727809e-07, + "loss": 0.0014, + "reward": 0.1171875, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1324 + }, + { + "completion_length": 121.65625, + "epoch": 0.7840236686390533, + "grad_norm": 1.038666847839775, + "kl": 0.031982421875, + "learning_rate": 2.1597633136094672e-07, + "loss": 0.0013, + "reward": 0.28125, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.28125, + "step": 1325 + }, + { + "completion_length": 124.890625, + "epoch": 0.7846153846153846, + "grad_norm": 0.7106373972792088, + "kl": 0.01953125, + "learning_rate": 2.153846153846154e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1326 + }, + { + "completion_length": 128.625, + "epoch": 0.785207100591716, + "grad_norm": 0.8009815064072245, + "kl": 0.031005859375, + "learning_rate": 2.1479289940828403e-07, + "loss": 0.0012, + "reward": 0.1953125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1327 + }, + { + "completion_length": 129.0859375, + "epoch": 0.7857988165680473, + "grad_norm": 0.5930006338654217, + "kl": 0.02099609375, + "learning_rate": 2.1420118343195266e-07, + "loss": 0.0008, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1328 + }, + { + "completion_length": 123.7890625, + "epoch": 0.7863905325443787, + "grad_norm": 0.7262289709951883, + "kl": 0.033203125, + "learning_rate": 2.136094674556213e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1329 + }, + { + "completion_length": 117.125, + "epoch": 0.7869822485207101, + "grad_norm": 0.6323374301901855, + "kl": 0.036376953125, + "learning_rate": 2.1301775147928995e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1330 + }, + { + "completion_length": 125.0, + "epoch": 0.7875739644970414, + "grad_norm": 0.5668981136721976, + "kl": 0.035400390625, + "learning_rate": 2.1242603550295858e-07, + "loss": 0.0014, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1331 + }, + { + "completion_length": 118.3125, + "epoch": 0.7881656804733728, + "grad_norm": 0.3044691305838001, + "kl": 0.0250244140625, + "learning_rate": 2.118343195266272e-07, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1332 + }, + { + "completion_length": 122.3515625, + "epoch": 0.7887573964497041, + "grad_norm": 0.7516343788790827, + "kl": 0.03125, + "learning_rate": 2.1124260355029586e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1333 + }, + { + "completion_length": 119.78125, + "epoch": 0.7893491124260354, + "grad_norm": 0.6168864747134698, + "kl": 0.03759765625, + "learning_rate": 2.106508875739645e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1334 + }, + { + "completion_length": 115.7578125, + "epoch": 0.7899408284023669, + "grad_norm": 0.5784750342787861, + "kl": 0.02880859375, + "learning_rate": 2.1005917159763312e-07, + "loss": 0.0012, + "reward": 0.1875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 1335 + }, + { + "completion_length": 125.453125, + "epoch": 0.7905325443786982, + "grad_norm": 0.703237520541177, + "kl": 0.045654296875, + "learning_rate": 2.0946745562130178e-07, + "loss": 0.0018, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1336 + }, + { + "completion_length": 121.4140625, + "epoch": 0.7911242603550296, + "grad_norm": 0.7813632546444146, + "kl": 0.0250244140625, + "learning_rate": 2.088757396449704e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1337 + }, + { + "completion_length": 125.265625, + "epoch": 0.7917159763313609, + "grad_norm": 0.6418940494289018, + "kl": 0.0308837890625, + "learning_rate": 2.0828402366863904e-07, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1338 + }, + { + "completion_length": 119.203125, + "epoch": 0.7923076923076923, + "grad_norm": 0.04939370479039887, + "kl": 0.01519775390625, + "learning_rate": 2.076923076923077e-07, + "loss": 0.0006, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1339 + }, + { + "completion_length": 125.0234375, + "epoch": 0.7928994082840237, + "grad_norm": 0.38579182457825956, + "kl": 0.0245361328125, + "learning_rate": 2.0710059171597633e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1340 + }, + { + "completion_length": 124.3671875, + "epoch": 0.7934911242603551, + "grad_norm": 0.3616912851655356, + "kl": 0.0230712890625, + "learning_rate": 2.0650887573964496e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1341 + }, + { + "completion_length": 126.2890625, + "epoch": 0.7940828402366864, + "grad_norm": 0.44322735143166025, + "kl": 0.0257568359375, + "learning_rate": 2.059171597633136e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1342 + }, + { + "completion_length": 116.2265625, + "epoch": 0.7946745562130177, + "grad_norm": 0.5133559361167451, + "kl": 0.0224609375, + "learning_rate": 2.0532544378698224e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1343 + }, + { + "completion_length": 117.296875, + "epoch": 0.7952662721893491, + "grad_norm": 0.6611060053813955, + "kl": 0.0296630859375, + "learning_rate": 2.0473372781065087e-07, + "loss": 0.0012, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1344 + }, + { + "completion_length": 122.3828125, + "epoch": 0.7958579881656804, + "grad_norm": 0.576907501939078, + "kl": 0.03173828125, + "learning_rate": 2.041420118343195e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1345 + }, + { + "completion_length": 128.8984375, + "epoch": 0.7964497041420119, + "grad_norm": 0.815270955159615, + "kl": 0.034912109375, + "learning_rate": 2.0355029585798816e-07, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1346 + }, + { + "completion_length": 126.046875, + "epoch": 0.7970414201183432, + "grad_norm": 0.47173905595738885, + "kl": 0.0218505859375, + "learning_rate": 2.029585798816568e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1347 + }, + { + "completion_length": 124.2734375, + "epoch": 0.7976331360946746, + "grad_norm": 0.8054400938640888, + "kl": 0.0242919921875, + "learning_rate": 2.0236686390532542e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1348 + }, + { + "completion_length": 128.8984375, + "epoch": 0.7982248520710059, + "grad_norm": 1.445474022294323, + "kl": 0.036376953125, + "learning_rate": 2.0177514792899407e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1349 + }, + { + "completion_length": 126.4375, + "epoch": 0.7988165680473372, + "grad_norm": 3.2280077351439593, + "kl": 0.03173828125, + "learning_rate": 2.011834319526627e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1350 + }, + { + "completion_length": 126.03125, + "epoch": 0.7994082840236686, + "grad_norm": 0.6631645770393344, + "kl": 0.026611328125, + "learning_rate": 2.0059171597633133e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1351 + }, + { + "completion_length": 132.8203125, + "epoch": 0.8, + "grad_norm": 0.3900139555491038, + "kl": 0.0255126953125, + "learning_rate": 2e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1352 + }, + { + "completion_length": 121.6640625, + "epoch": 0.8005917159763314, + "grad_norm": 0.5554775649538611, + "kl": 0.037109375, + "learning_rate": 1.9940828402366862e-07, + "loss": 0.0015, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1353 + }, + { + "completion_length": 121.59375, + "epoch": 0.8011834319526627, + "grad_norm": 0.3062770925970396, + "kl": 0.01904296875, + "learning_rate": 1.9881656804733725e-07, + "loss": 0.0008, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1354 + }, + { + "completion_length": 116.1015625, + "epoch": 0.8017751479289941, + "grad_norm": 0.8987441775615598, + "kl": 0.044677734375, + "learning_rate": 1.9822485207100593e-07, + "loss": 0.0018, + "reward": 0.1171875, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1355 + }, + { + "completion_length": 126.6875, + "epoch": 0.8023668639053254, + "grad_norm": 0.27927025098335084, + "kl": 0.0191650390625, + "learning_rate": 1.9763313609467456e-07, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1356 + }, + { + "completion_length": 127.875, + "epoch": 0.8029585798816568, + "grad_norm": 0.6055683382724127, + "kl": 0.021240234375, + "learning_rate": 1.970414201183432e-07, + "loss": 0.0008, + "reward": 0.0625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1357 + }, + { + "completion_length": 131.1953125, + "epoch": 0.8035502958579882, + "grad_norm": 0.6161418250883507, + "kl": 0.03955078125, + "learning_rate": 1.9644970414201182e-07, + "loss": 0.0016, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1358 + }, + { + "completion_length": 130.7578125, + "epoch": 0.8041420118343195, + "grad_norm": 0.32932021452386184, + "kl": 0.0223388671875, + "learning_rate": 1.9585798816568048e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1359 + }, + { + "completion_length": 121.0390625, + "epoch": 0.8047337278106509, + "grad_norm": 0.3755349184574648, + "kl": 0.0263671875, + "learning_rate": 1.952662721893491e-07, + "loss": 0.0011, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1360 + }, + { + "completion_length": 122.8515625, + "epoch": 0.8053254437869822, + "grad_norm": 0.3091587572752453, + "kl": 0.032958984375, + "learning_rate": 1.9467455621301774e-07, + "loss": 0.0013, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1361 + }, + { + "completion_length": 127.8515625, + "epoch": 0.8059171597633136, + "grad_norm": 0.6052302081360669, + "kl": 0.0274658203125, + "learning_rate": 1.940828402366864e-07, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1362 + }, + { + "completion_length": 125.09375, + "epoch": 0.806508875739645, + "grad_norm": 0.3350698325069941, + "kl": 0.0211181640625, + "learning_rate": 1.9349112426035503e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1363 + }, + { + "completion_length": 124.828125, + "epoch": 0.8071005917159764, + "grad_norm": 0.4843092803509769, + "kl": 0.03076171875, + "learning_rate": 1.9289940828402366e-07, + "loss": 0.0012, + "reward": 0.1875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 1364 + }, + { + "completion_length": 126.0859375, + "epoch": 0.8076923076923077, + "grad_norm": 6.689934148941216, + "kl": 0.0294189453125, + "learning_rate": 1.9230769230769231e-07, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1365 + }, + { + "completion_length": 135.25, + "epoch": 0.808284023668639, + "grad_norm": 0.8047631024214, + "kl": 0.02490234375, + "learning_rate": 1.9171597633136094e-07, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1366 + }, + { + "completion_length": 120.296875, + "epoch": 0.8088757396449704, + "grad_norm": 0.38475279173155424, + "kl": 0.03955078125, + "learning_rate": 1.9112426035502957e-07, + "loss": 0.0016, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1367 + }, + { + "completion_length": 122.90625, + "epoch": 0.8094674556213017, + "grad_norm": 0.6151253595649567, + "kl": 0.0269775390625, + "learning_rate": 1.9053254437869823e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1368 + }, + { + "completion_length": 125.1328125, + "epoch": 0.8100591715976332, + "grad_norm": 0.35351377803259443, + "kl": 0.0240478515625, + "learning_rate": 1.8994082840236686e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1369 + }, + { + "completion_length": 119.9140625, + "epoch": 0.8106508875739645, + "grad_norm": 0.5388680402887523, + "kl": 0.0234375, + "learning_rate": 1.893491124260355e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1370 + }, + { + "completion_length": 126.9921875, + "epoch": 0.8112426035502959, + "grad_norm": 0.580779151375869, + "kl": 0.031005859375, + "learning_rate": 1.8875739644970412e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1371 + }, + { + "completion_length": 127.7421875, + "epoch": 0.8118343195266272, + "grad_norm": 0.38799442262099776, + "kl": 0.0208740234375, + "learning_rate": 1.8816568047337278e-07, + "loss": 0.0008, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1372 + }, + { + "completion_length": 133.8515625, + "epoch": 0.8124260355029586, + "grad_norm": 1.2194091274395567, + "kl": 0.025634765625, + "learning_rate": 1.875739644970414e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1373 + }, + { + "completion_length": 121.1015625, + "epoch": 0.8130177514792899, + "grad_norm": 0.6014253484100058, + "kl": 0.0439453125, + "learning_rate": 1.8698224852071004e-07, + "loss": 0.0018, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1374 + }, + { + "completion_length": 124.15625, + "epoch": 0.8136094674556213, + "grad_norm": 0.7223723145793034, + "kl": 0.0322265625, + "learning_rate": 1.863905325443787e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1375 + }, + { + "completion_length": 122.71875, + "epoch": 0.8142011834319527, + "grad_norm": 1.8989816825128365, + "kl": 0.0264892578125, + "learning_rate": 1.8579881656804732e-07, + "loss": 0.0011, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1376 + }, + { + "completion_length": 118.2734375, + "epoch": 0.814792899408284, + "grad_norm": 0.26895094133991737, + "kl": 0.023193359375, + "learning_rate": 1.8520710059171595e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1377 + }, + { + "completion_length": 126.2890625, + "epoch": 0.8153846153846154, + "grad_norm": 0.08052415095695203, + "kl": 0.0213623046875, + "learning_rate": 1.846153846153846e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1378 + }, + { + "completion_length": 121.796875, + "epoch": 0.8159763313609467, + "grad_norm": 0.389954922435328, + "kl": 0.021728515625, + "learning_rate": 1.8402366863905324e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1379 + }, + { + "completion_length": 122.5703125, + "epoch": 0.8165680473372781, + "grad_norm": 0.4957687707172047, + "kl": 0.030517578125, + "learning_rate": 1.8343195266272187e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1380 + }, + { + "completion_length": 121.7890625, + "epoch": 0.8171597633136095, + "grad_norm": 0.22883606260943623, + "kl": 0.020751953125, + "learning_rate": 1.8284023668639053e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1381 + }, + { + "completion_length": 129.4921875, + "epoch": 0.8177514792899409, + "grad_norm": 0.553383672757578, + "kl": 0.0228271484375, + "learning_rate": 1.8224852071005916e-07, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1382 + }, + { + "completion_length": 117.6015625, + "epoch": 0.8183431952662722, + "grad_norm": 0.5880021521154032, + "kl": 0.0250244140625, + "learning_rate": 1.8165680473372779e-07, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1383 + }, + { + "completion_length": 118.09375, + "epoch": 0.8189349112426035, + "grad_norm": 0.4732612456239916, + "kl": 0.0264892578125, + "learning_rate": 1.8106508875739642e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1384 + }, + { + "completion_length": 119.75, + "epoch": 0.8195266272189349, + "grad_norm": 0.43710675925088494, + "kl": 0.020263671875, + "learning_rate": 1.804733727810651e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1385 + }, + { + "completion_length": 121.8515625, + "epoch": 0.8201183431952662, + "grad_norm": 0.41892319664241556, + "kl": 0.0279541015625, + "learning_rate": 1.7988165680473373e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1386 + }, + { + "completion_length": 120.8125, + "epoch": 0.8207100591715977, + "grad_norm": 0.7168342819365654, + "kl": 0.03857421875, + "learning_rate": 1.7928994082840236e-07, + "loss": 0.0015, + "reward": 0.0546875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1387 + }, + { + "completion_length": 118.9609375, + "epoch": 0.821301775147929, + "grad_norm": 0.3643962301892449, + "kl": 0.018798828125, + "learning_rate": 1.7869822485207101e-07, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1388 + }, + { + "completion_length": 113.75, + "epoch": 0.8218934911242604, + "grad_norm": 0.5612096150264649, + "kl": 0.03173828125, + "learning_rate": 1.7810650887573964e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1389 + }, + { + "completion_length": 117.28125, + "epoch": 0.8224852071005917, + "grad_norm": 0.5661476198496381, + "kl": 0.026611328125, + "learning_rate": 1.7751479289940827e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1390 + }, + { + "completion_length": 120.6484375, + "epoch": 0.823076923076923, + "grad_norm": 0.6733887417356951, + "kl": 0.03759765625, + "learning_rate": 1.7692307692307693e-07, + "loss": 0.0015, + "reward": 0.2265625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2265625, + "step": 1391 + }, + { + "completion_length": 118.3984375, + "epoch": 0.8236686390532545, + "grad_norm": 0.5005086283657637, + "kl": 0.02294921875, + "learning_rate": 1.7633136094674556e-07, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1392 + }, + { + "completion_length": 129.9375, + "epoch": 0.8242603550295858, + "grad_norm": 0.7508728354555555, + "kl": 0.0291748046875, + "learning_rate": 1.757396449704142e-07, + "loss": 0.0012, + "reward": 0.1640625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1393 + }, + { + "completion_length": 120.0546875, + "epoch": 0.8248520710059172, + "grad_norm": 0.3371469194176043, + "kl": 0.0322265625, + "learning_rate": 1.7514792899408285e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1394 + }, + { + "completion_length": 113.421875, + "epoch": 0.8254437869822485, + "grad_norm": 0.72734830601016, + "kl": 0.0439453125, + "learning_rate": 1.7455621301775148e-07, + "loss": 0.0018, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1395 + }, + { + "completion_length": 107.546875, + "epoch": 0.8260355029585799, + "grad_norm": 0.6056522181411265, + "kl": 0.0322265625, + "learning_rate": 1.739644970414201e-07, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1396 + }, + { + "completion_length": 121.25, + "epoch": 0.8266272189349112, + "grad_norm": 0.7089618594295822, + "kl": 0.030517578125, + "learning_rate": 1.7337278106508874e-07, + "loss": 0.0012, + "reward": 0.15625, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1397 + }, + { + "completion_length": 117.375, + "epoch": 0.8272189349112427, + "grad_norm": 0.530066258258225, + "kl": 0.0233154296875, + "learning_rate": 1.727810650887574e-07, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0078125, + "rewards/message_judge_reward": 0.1015625, + "step": 1398 + }, + { + "completion_length": 122.5546875, + "epoch": 0.827810650887574, + "grad_norm": 0.47108384440585255, + "kl": 0.0322265625, + "learning_rate": 1.7218934911242602e-07, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1399 + }, + { + "completion_length": 129.8984375, + "epoch": 0.8284023668639053, + "grad_norm": 0.6636802236362482, + "kl": 0.03759765625, + "learning_rate": 1.7159763313609465e-07, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1400 + }, + { + "completion_length": 118.828125, + "epoch": 0.8289940828402367, + "grad_norm": 0.2741658754922165, + "kl": 0.0181884765625, + "learning_rate": 1.710059171597633e-07, + "loss": 0.0007, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1401 + }, + { + "completion_length": 124.921875, + "epoch": 0.829585798816568, + "grad_norm": 0.6108373837924167, + "kl": 0.03271484375, + "learning_rate": 1.7041420118343194e-07, + "loss": 0.0013, + "reward": 0.2109375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2109375, + "step": 1402 + }, + { + "completion_length": 118.2109375, + "epoch": 0.8301775147928994, + "grad_norm": 0.34389073207528215, + "kl": 0.039794921875, + "learning_rate": 1.6982248520710057e-07, + "loss": 0.0016, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1403 + }, + { + "completion_length": 117.1328125, + "epoch": 0.8307692307692308, + "grad_norm": 0.7268281123601918, + "kl": 0.04345703125, + "learning_rate": 1.6923076923076923e-07, + "loss": 0.0017, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1404 + }, + { + "completion_length": 128.5703125, + "epoch": 0.8313609467455622, + "grad_norm": 0.42991138771247656, + "kl": 0.03515625, + "learning_rate": 1.6863905325443786e-07, + "loss": 0.0014, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1405 + }, + { + "completion_length": 114.0390625, + "epoch": 0.8319526627218935, + "grad_norm": 1.3449241375751138, + "kl": 0.040771484375, + "learning_rate": 1.680473372781065e-07, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1406 + }, + { + "completion_length": 123.03125, + "epoch": 0.8325443786982248, + "grad_norm": 0.6075677145410301, + "kl": 0.030517578125, + "learning_rate": 1.6745562130177514e-07, + "loss": 0.0012, + "reward": 0.15625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1407 + }, + { + "completion_length": 124.2890625, + "epoch": 0.8331360946745562, + "grad_norm": 0.5435526201356595, + "kl": 0.0284423828125, + "learning_rate": 1.6686390532544377e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1408 + }, + { + "completion_length": 116.3125, + "epoch": 0.8337278106508875, + "grad_norm": 0.5990625944056855, + "kl": 0.0299072265625, + "learning_rate": 1.662721893491124e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1409 + }, + { + "completion_length": 127.0390625, + "epoch": 0.834319526627219, + "grad_norm": 0.47314062167694804, + "kl": 0.0230712890625, + "learning_rate": 1.6568047337278106e-07, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1410 + }, + { + "completion_length": 115.125, + "epoch": 0.8349112426035503, + "grad_norm": 0.1582137683219498, + "kl": 0.0263671875, + "learning_rate": 1.650887573964497e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1411 + }, + { + "completion_length": 121.0625, + "epoch": 0.8355029585798817, + "grad_norm": 0.4980086115477253, + "kl": 0.0205078125, + "learning_rate": 1.6449704142011832e-07, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1412 + }, + { + "completion_length": 128.1796875, + "epoch": 0.836094674556213, + "grad_norm": 0.296576606920655, + "kl": 0.0191650390625, + "learning_rate": 1.6390532544378695e-07, + "loss": 0.0008, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1413 + }, + { + "completion_length": 135.640625, + "epoch": 0.8366863905325443, + "grad_norm": 0.4672849604783081, + "kl": 0.0269775390625, + "learning_rate": 1.6331360946745563e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1414 + }, + { + "completion_length": 115.3125, + "epoch": 0.8372781065088757, + "grad_norm": 0.4084554764442433, + "kl": 0.03271484375, + "learning_rate": 1.6272189349112426e-07, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1415 + }, + { + "completion_length": 119.8125, + "epoch": 0.8378698224852071, + "grad_norm": 0.427809456074547, + "kl": 0.027587890625, + "learning_rate": 1.621301775147929e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1416 + }, + { + "completion_length": 121.3203125, + "epoch": 0.8384615384615385, + "grad_norm": 0.44736503788622506, + "kl": 0.036376953125, + "learning_rate": 1.6153846153846155e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1417 + }, + { + "completion_length": 118.8515625, + "epoch": 0.8390532544378698, + "grad_norm": 0.5411597902712657, + "kl": 0.0247802734375, + "learning_rate": 1.6094674556213018e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1418 + }, + { + "completion_length": 117.1640625, + "epoch": 0.8396449704142012, + "grad_norm": 0.7583778784130998, + "kl": 0.045166015625, + "learning_rate": 1.603550295857988e-07, + "loss": 0.0018, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1419 + }, + { + "completion_length": 117.515625, + "epoch": 0.8402366863905325, + "grad_norm": 0.4008677865643227, + "kl": 0.0262451171875, + "learning_rate": 1.5976331360946747e-07, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1420 + }, + { + "completion_length": 120.8125, + "epoch": 0.840828402366864, + "grad_norm": 0.7498961676262424, + "kl": 0.026123046875, + "learning_rate": 1.591715976331361e-07, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1421 + }, + { + "completion_length": 123.5390625, + "epoch": 0.8414201183431953, + "grad_norm": 0.6176621185164701, + "kl": 0.0277099609375, + "learning_rate": 1.5857988165680473e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1422 + }, + { + "completion_length": 125.71875, + "epoch": 0.8420118343195266, + "grad_norm": 0.06799901536501597, + "kl": 0.0235595703125, + "learning_rate": 1.5798816568047338e-07, + "loss": 0.0009, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1423 + }, + { + "completion_length": 126.4453125, + "epoch": 0.842603550295858, + "grad_norm": 0.36428107660872344, + "kl": 0.0322265625, + "learning_rate": 1.57396449704142e-07, + "loss": 0.0013, + "reward": 0.1953125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1424 + }, + { + "completion_length": 116.5625, + "epoch": 0.8431952662721893, + "grad_norm": 0.5240099773607524, + "kl": 0.035400390625, + "learning_rate": 1.5680473372781064e-07, + "loss": 0.0014, + "reward": 0.15625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1425 + }, + { + "completion_length": 119.390625, + "epoch": 0.8437869822485207, + "grad_norm": 0.40794343244268744, + "kl": 0.0302734375, + "learning_rate": 1.5621301775147927e-07, + "loss": 0.0012, + "reward": 0.125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1426 + }, + { + "completion_length": 118.875, + "epoch": 0.8443786982248521, + "grad_norm": 0.30310907143675714, + "kl": 0.021240234375, + "learning_rate": 1.5562130177514793e-07, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1427 + }, + { + "completion_length": 112.71875, + "epoch": 0.8449704142011835, + "grad_norm": 0.3936509455625125, + "kl": 0.02880859375, + "learning_rate": 1.5502958579881656e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1428 + }, + { + "completion_length": 129.8359375, + "epoch": 0.8455621301775148, + "grad_norm": 0.7130436975683455, + "kl": 0.0218505859375, + "learning_rate": 1.544378698224852e-07, + "loss": 0.0009, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1429 + }, + { + "completion_length": 119.78125, + "epoch": 0.8461538461538461, + "grad_norm": 0.4230193494998163, + "kl": 0.034912109375, + "learning_rate": 1.5384615384615385e-07, + "loss": 0.0014, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1430 + }, + { + "completion_length": 127.8125, + "epoch": 0.8467455621301775, + "grad_norm": 0.4112968369035015, + "kl": 0.0291748046875, + "learning_rate": 1.5325443786982248e-07, + "loss": 0.0012, + "reward": 0.015625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1431 + }, + { + "completion_length": 122.65625, + "epoch": 0.8473372781065088, + "grad_norm": 0.05431543364636355, + "kl": 0.0269775390625, + "learning_rate": 1.526627218934911e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1432 + }, + { + "completion_length": 117.890625, + "epoch": 0.8479289940828403, + "grad_norm": 1.6870335215277752, + "kl": 0.03271484375, + "learning_rate": 1.5207100591715976e-07, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1433 + }, + { + "completion_length": 124.1640625, + "epoch": 0.8485207100591716, + "grad_norm": 0.6284037908150184, + "kl": 0.0230712890625, + "learning_rate": 1.514792899408284e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1434 + }, + { + "completion_length": 130.125, + "epoch": 0.849112426035503, + "grad_norm": 1.4846781696784643, + "kl": 0.02587890625, + "learning_rate": 1.5088757396449702e-07, + "loss": 0.001, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1435 + }, + { + "completion_length": 128.578125, + "epoch": 0.8497041420118343, + "grad_norm": 0.05962336628687922, + "kl": 0.0255126953125, + "learning_rate": 1.5029585798816568e-07, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1436 + }, + { + "completion_length": 112.53125, + "epoch": 0.8502958579881656, + "grad_norm": 0.5561433151951933, + "kl": 0.033203125, + "learning_rate": 1.497041420118343e-07, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1437 + }, + { + "completion_length": 124.28125, + "epoch": 0.850887573964497, + "grad_norm": 0.44744200274703233, + "kl": 0.0269775390625, + "learning_rate": 1.4911242603550294e-07, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1438 + }, + { + "completion_length": 122.0625, + "epoch": 0.8514792899408284, + "grad_norm": 0.4843192792521466, + "kl": 0.02392578125, + "learning_rate": 1.4852071005917157e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1439 + }, + { + "completion_length": 120.9765625, + "epoch": 0.8520710059171598, + "grad_norm": 0.43898589627881535, + "kl": 0.03125, + "learning_rate": 1.4792899408284022e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1440 + }, + { + "completion_length": 115.234375, + "epoch": 0.8526627218934911, + "grad_norm": 1.5098360073235169, + "kl": 0.036865234375, + "learning_rate": 1.4733727810650885e-07, + "loss": 0.0015, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1441 + }, + { + "completion_length": 127.34375, + "epoch": 0.8532544378698225, + "grad_norm": 0.4979281583271304, + "kl": 0.03466796875, + "learning_rate": 1.4674556213017748e-07, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1442 + }, + { + "completion_length": 121.953125, + "epoch": 0.8538461538461538, + "grad_norm": 0.6708374140612932, + "kl": 0.03564453125, + "learning_rate": 1.4615384615384617e-07, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1443 + }, + { + "completion_length": 118.5546875, + "epoch": 0.8544378698224852, + "grad_norm": 0.4054576283798711, + "kl": 0.02197265625, + "learning_rate": 1.455621301775148e-07, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1444 + }, + { + "completion_length": 122.828125, + "epoch": 0.8550295857988166, + "grad_norm": 0.5793993881956386, + "kl": 0.0306396484375, + "learning_rate": 1.4497041420118343e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1445 + }, + { + "completion_length": 113.2109375, + "epoch": 0.855621301775148, + "grad_norm": 0.5344704029129015, + "kl": 0.02880859375, + "learning_rate": 1.4437869822485208e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1446 + }, + { + "completion_length": 119.0, + "epoch": 0.8562130177514793, + "grad_norm": 0.45238976991430185, + "kl": 0.0242919921875, + "learning_rate": 1.4378698224852071e-07, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1447 + }, + { + "completion_length": 126.015625, + "epoch": 0.8568047337278106, + "grad_norm": 0.9080078172008128, + "kl": 0.0235595703125, + "learning_rate": 1.4319526627218934e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1448 + }, + { + "completion_length": 117.3984375, + "epoch": 0.857396449704142, + "grad_norm": 0.41754324422429706, + "kl": 0.0311279296875, + "learning_rate": 1.42603550295858e-07, + "loss": 0.0012, + "reward": 0.1640625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1449 + }, + { + "completion_length": 126.3125, + "epoch": 0.8579881656804734, + "grad_norm": 0.6825819373969387, + "kl": 0.0302734375, + "learning_rate": 1.4201183431952663e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1450 + }, + { + "completion_length": 137.0625, + "epoch": 0.8585798816568048, + "grad_norm": 0.11942984054582828, + "kl": 0.0260009765625, + "learning_rate": 1.4142011834319526e-07, + "loss": 0.001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1451 + }, + { + "completion_length": 116.6015625, + "epoch": 0.8591715976331361, + "grad_norm": 0.7039485413807578, + "kl": 0.0283203125, + "learning_rate": 1.408284023668639e-07, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1452 + }, + { + "completion_length": 126.109375, + "epoch": 0.8597633136094674, + "grad_norm": 0.732686547634708, + "kl": 0.04052734375, + "learning_rate": 1.4023668639053255e-07, + "loss": 0.0016, + "reward": 0.21875, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.21875, + "step": 1453 + }, + { + "completion_length": 122.84375, + "epoch": 0.8603550295857988, + "grad_norm": 0.7992257369531476, + "kl": 0.026123046875, + "learning_rate": 1.3964497041420118e-07, + "loss": 0.001, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1454 + }, + { + "completion_length": 119.6171875, + "epoch": 0.8609467455621301, + "grad_norm": 0.805855154545763, + "kl": 0.0322265625, + "learning_rate": 1.390532544378698e-07, + "loss": 0.0013, + "reward": 0.203125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.203125, + "step": 1455 + }, + { + "completion_length": 122.09375, + "epoch": 0.8615384615384616, + "grad_norm": 0.6953130228632028, + "kl": 0.033203125, + "learning_rate": 1.3846153846153846e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1456 + }, + { + "completion_length": 116.4453125, + "epoch": 0.8621301775147929, + "grad_norm": 0.35217308705050027, + "kl": 0.0283203125, + "learning_rate": 1.378698224852071e-07, + "loss": 0.0011, + "reward": 0.140625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1457 + }, + { + "completion_length": 124.3828125, + "epoch": 0.8627218934911243, + "grad_norm": 0.8006144002556724, + "kl": 0.030029296875, + "learning_rate": 1.3727810650887572e-07, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1458 + }, + { + "completion_length": 118.9375, + "epoch": 0.8633136094674556, + "grad_norm": 0.3962285458205043, + "kl": 0.0277099609375, + "learning_rate": 1.3668639053254438e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1459 + }, + { + "completion_length": 128.6875, + "epoch": 0.863905325443787, + "grad_norm": 0.5204792004628169, + "kl": 0.0302734375, + "learning_rate": 1.36094674556213e-07, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1460 + }, + { + "completion_length": 121.265625, + "epoch": 0.8644970414201183, + "grad_norm": 0.2916309020008978, + "kl": 0.0238037109375, + "learning_rate": 1.3550295857988164e-07, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1461 + }, + { + "completion_length": 128.1796875, + "epoch": 0.8650887573964497, + "grad_norm": 0.37609068102768456, + "kl": 0.026611328125, + "learning_rate": 1.349112426035503e-07, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1462 + }, + { + "completion_length": 124.109375, + "epoch": 0.8656804733727811, + "grad_norm": 0.050780095566048546, + "kl": 0.03173828125, + "learning_rate": 1.3431952662721893e-07, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1463 + }, + { + "completion_length": 125.6796875, + "epoch": 0.8662721893491124, + "grad_norm": 0.510618054093859, + "kl": 0.029296875, + "learning_rate": 1.3372781065088756e-07, + "loss": 0.0012, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1464 + }, + { + "completion_length": 124.53125, + "epoch": 0.8668639053254438, + "grad_norm": 0.8345344807553834, + "kl": 0.034423828125, + "learning_rate": 1.331360946745562e-07, + "loss": 0.0014, + "reward": 0.171875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1465 + }, + { + "completion_length": 120.40625, + "epoch": 0.8674556213017751, + "grad_norm": 0.5543346814511463, + "kl": 0.037353515625, + "learning_rate": 1.3254437869822484e-07, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1466 + }, + { + "completion_length": 119.8671875, + "epoch": 0.8680473372781065, + "grad_norm": 0.8210562958449976, + "kl": 0.030517578125, + "learning_rate": 1.3195266272189347e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1467 + }, + { + "completion_length": 118.671875, + "epoch": 0.8686390532544379, + "grad_norm": 0.6531107949098011, + "kl": 0.03271484375, + "learning_rate": 1.313609467455621e-07, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1468 + }, + { + "completion_length": 120.40625, + "epoch": 0.8692307692307693, + "grad_norm": 0.4565831216486855, + "kl": 0.021728515625, + "learning_rate": 1.3076923076923076e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1469 + }, + { + "completion_length": 115.3125, + "epoch": 0.8698224852071006, + "grad_norm": 0.3744547394406036, + "kl": 0.0247802734375, + "learning_rate": 1.301775147928994e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1470 + }, + { + "completion_length": 122.359375, + "epoch": 0.8704142011834319, + "grad_norm": 0.8469035269173799, + "kl": 0.041015625, + "learning_rate": 1.2958579881656802e-07, + "loss": 0.0016, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1471 + }, + { + "completion_length": 129.9609375, + "epoch": 0.8710059171597633, + "grad_norm": 0.39310000997994954, + "kl": 0.0240478515625, + "learning_rate": 1.289940828402367e-07, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1472 + }, + { + "completion_length": 122.515625, + "epoch": 0.8715976331360947, + "grad_norm": 0.615428552175192, + "kl": 0.026123046875, + "learning_rate": 1.2840236686390533e-07, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1473 + }, + { + "completion_length": 126.109375, + "epoch": 0.8721893491124261, + "grad_norm": 0.4100447211396515, + "kl": 0.031005859375, + "learning_rate": 1.2781065088757396e-07, + "loss": 0.0012, + "reward": 0.171875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1474 + }, + { + "completion_length": 125.1328125, + "epoch": 0.8727810650887574, + "grad_norm": 0.6118213349829504, + "kl": 0.025146484375, + "learning_rate": 1.2721893491124262e-07, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1475 + }, + { + "completion_length": 112.6328125, + "epoch": 0.8733727810650888, + "grad_norm": 0.577318321568965, + "kl": 0.039794921875, + "learning_rate": 1.2662721893491125e-07, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1476 + }, + { + "completion_length": 128.9140625, + "epoch": 0.8739644970414201, + "grad_norm": 0.37280822546685594, + "kl": 0.0296630859375, + "learning_rate": 1.2603550295857988e-07, + "loss": 0.0012, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1477 + }, + { + "completion_length": 115.6953125, + "epoch": 0.8745562130177514, + "grad_norm": 0.6142835367355567, + "kl": 0.026611328125, + "learning_rate": 1.2544378698224854e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1478 + }, + { + "completion_length": 123.421875, + "epoch": 0.8751479289940829, + "grad_norm": 0.5385774323724932, + "kl": 0.036376953125, + "learning_rate": 1.2485207100591717e-07, + "loss": 0.0015, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1479 + }, + { + "completion_length": 115.6328125, + "epoch": 0.8757396449704142, + "grad_norm": 0.2884275015123521, + "kl": 0.023681640625, + "learning_rate": 1.242603550295858e-07, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1480 + }, + { + "completion_length": 111.8984375, + "epoch": 0.8763313609467456, + "grad_norm": 1.1456426814200584, + "kl": 0.041015625, + "learning_rate": 1.2366863905325443e-07, + "loss": 0.0016, + "reward": 0.2421875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.2421875, + "step": 1481 + }, + { + "completion_length": 111.015625, + "epoch": 0.8769230769230769, + "grad_norm": 0.7014303884708996, + "kl": 0.03515625, + "learning_rate": 1.2307692307692308e-07, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1482 + }, + { + "completion_length": 126.3203125, + "epoch": 0.8775147928994083, + "grad_norm": 0.5199769854644236, + "kl": 0.02978515625, + "learning_rate": 1.224852071005917e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1483 + }, + { + "completion_length": 118.625, + "epoch": 0.8781065088757396, + "grad_norm": 0.7229426726068398, + "kl": 0.03271484375, + "learning_rate": 1.2189349112426034e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1484 + }, + { + "completion_length": 123.5, + "epoch": 0.878698224852071, + "grad_norm": 0.37186760540067965, + "kl": 0.02001953125, + "learning_rate": 1.2130177514792897e-07, + "loss": 0.0008, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1485 + }, + { + "completion_length": 122.84375, + "epoch": 0.8792899408284024, + "grad_norm": 0.7724669656796185, + "kl": 0.031005859375, + "learning_rate": 1.2071005917159763e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1486 + }, + { + "completion_length": 112.9765625, + "epoch": 0.8798816568047337, + "grad_norm": 0.7693398398951938, + "kl": 0.036865234375, + "learning_rate": 1.2011834319526628e-07, + "loss": 0.0015, + "reward": 0.1640625, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1487 + }, + { + "completion_length": 122.4921875, + "epoch": 0.8804733727810651, + "grad_norm": 0.7123162383458107, + "kl": 0.038330078125, + "learning_rate": 1.1952662721893491e-07, + "loss": 0.0015, + "reward": 0.09375, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1488 + }, + { + "completion_length": 111.2890625, + "epoch": 0.8810650887573964, + "grad_norm": 0.7704492043512882, + "kl": 0.037841796875, + "learning_rate": 1.1893491124260354e-07, + "loss": 0.0015, + "reward": 0.1328125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1489 + }, + { + "completion_length": 114.78125, + "epoch": 0.8816568047337278, + "grad_norm": 0.6594908608783109, + "kl": 0.030029296875, + "learning_rate": 1.1834319526627217e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1490 + }, + { + "completion_length": 118.8046875, + "epoch": 0.8822485207100592, + "grad_norm": 0.6057743193250671, + "kl": 0.02734375, + "learning_rate": 1.1775147928994082e-07, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1491 + }, + { + "completion_length": 112.375, + "epoch": 0.8828402366863906, + "grad_norm": 0.820531418944025, + "kl": 0.030517578125, + "learning_rate": 1.1715976331360946e-07, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.09858439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1492 + }, + { + "completion_length": 119.9140625, + "epoch": 0.8834319526627219, + "grad_norm": 0.6754507916250675, + "kl": 0.0299072265625, + "learning_rate": 1.1656804733727809e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1493 + }, + { + "completion_length": 123.359375, + "epoch": 0.8840236686390532, + "grad_norm": 0.6395157478661595, + "kl": 0.0302734375, + "learning_rate": 1.1597633136094675e-07, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1494 + }, + { + "completion_length": 114.25, + "epoch": 0.8846153846153846, + "grad_norm": 0.29235917509517406, + "kl": 0.034423828125, + "learning_rate": 1.1538461538461539e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1495 + }, + { + "completion_length": 109.375, + "epoch": 0.8852071005917159, + "grad_norm": 0.5318108772048202, + "kl": 0.033203125, + "learning_rate": 1.1479289940828402e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1496 + }, + { + "completion_length": 122.3046875, + "epoch": 0.8857988165680474, + "grad_norm": 0.4956649430558182, + "kl": 0.026611328125, + "learning_rate": 1.1420118343195266e-07, + "loss": 0.0011, + "reward": 0.1953125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1497 + }, + { + "completion_length": 123.8828125, + "epoch": 0.8863905325443787, + "grad_norm": 0.6506125667997134, + "kl": 0.0218505859375, + "learning_rate": 1.136094674556213e-07, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1498 + }, + { + "completion_length": 118.203125, + "epoch": 0.8869822485207101, + "grad_norm": 0.2196316662743563, + "kl": 0.0228271484375, + "learning_rate": 1.1301775147928994e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1499 + }, + { + "completion_length": 111.5078125, + "epoch": 0.8875739644970414, + "grad_norm": 0.498094332822964, + "kl": 0.0361328125, + "learning_rate": 1.1242603550295858e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1500 + }, + { + "completion_length": 116.609375, + "epoch": 0.8881656804733727, + "grad_norm": 0.3414388509521559, + "kl": 0.0281982421875, + "learning_rate": 1.1183431952662721e-07, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1501 + }, + { + "completion_length": 125.046875, + "epoch": 0.8887573964497042, + "grad_norm": 0.3360323636938933, + "kl": 0.0185546875, + "learning_rate": 1.1124260355029585e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1502 + }, + { + "completion_length": 116.8671875, + "epoch": 0.8893491124260355, + "grad_norm": 0.45365185725438695, + "kl": 0.0380859375, + "learning_rate": 1.1065088757396448e-07, + "loss": 0.0015, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1503 + }, + { + "completion_length": 118.28125, + "epoch": 0.8899408284023669, + "grad_norm": 0.504506852192402, + "kl": 0.0159912109375, + "learning_rate": 1.1005917159763313e-07, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1504 + }, + { + "completion_length": 111.46875, + "epoch": 0.8905325443786982, + "grad_norm": 0.6754222678934975, + "kl": 0.0306396484375, + "learning_rate": 1.0946745562130177e-07, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1505 + }, + { + "completion_length": 109.296875, + "epoch": 0.8911242603550296, + "grad_norm": 0.6255427882889124, + "kl": 0.037109375, + "learning_rate": 1.088757396449704e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1506 + }, + { + "completion_length": 121.1640625, + "epoch": 0.8917159763313609, + "grad_norm": 0.6012807430894747, + "kl": 0.033447265625, + "learning_rate": 1.0828402366863904e-07, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1507 + }, + { + "completion_length": 115.453125, + "epoch": 0.8923076923076924, + "grad_norm": 0.07678427134259139, + "kl": 0.03759765625, + "learning_rate": 1.076923076923077e-07, + "loss": 0.0015, + "reward": 0.125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1508 + }, + { + "completion_length": 115.921875, + "epoch": 0.8928994082840237, + "grad_norm": 0.3149219938039627, + "kl": 0.035888671875, + "learning_rate": 1.0710059171597633e-07, + "loss": 0.0014, + "reward": 0.1796875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 1509 + }, + { + "completion_length": 122.3671875, + "epoch": 0.893491124260355, + "grad_norm": 0.3886145916517842, + "kl": 0.02783203125, + "learning_rate": 1.0650887573964497e-07, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1510 + }, + { + "completion_length": 117.7265625, + "epoch": 0.8940828402366864, + "grad_norm": 0.2530689902792644, + "kl": 0.02587890625, + "learning_rate": 1.059171597633136e-07, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1511 + }, + { + "completion_length": 125.078125, + "epoch": 0.8946745562130177, + "grad_norm": 0.23794122225619851, + "kl": 0.0224609375, + "learning_rate": 1.0532544378698225e-07, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1512 + }, + { + "completion_length": 122.578125, + "epoch": 0.8952662721893491, + "grad_norm": 0.603445978257032, + "kl": 0.03466796875, + "learning_rate": 1.0473372781065089e-07, + "loss": 0.0014, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1513 + }, + { + "completion_length": 121.140625, + "epoch": 0.8958579881656805, + "grad_norm": 0.5042495836634966, + "kl": 0.03271484375, + "learning_rate": 1.0414201183431952e-07, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1514 + }, + { + "completion_length": 115.3984375, + "epoch": 0.8964497041420119, + "grad_norm": 0.5953286777945979, + "kl": 0.030029296875, + "learning_rate": 1.0355029585798816e-07, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1515 + }, + { + "completion_length": 122.4921875, + "epoch": 0.8970414201183432, + "grad_norm": 0.5247165824631262, + "kl": 0.037841796875, + "learning_rate": 1.029585798816568e-07, + "loss": 0.0015, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1516 + }, + { + "completion_length": 126.125, + "epoch": 0.8976331360946745, + "grad_norm": 0.2624611461737152, + "kl": 0.0185546875, + "learning_rate": 1.0236686390532544e-07, + "loss": 0.0007, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1517 + }, + { + "completion_length": 110.3671875, + "epoch": 0.8982248520710059, + "grad_norm": 0.5019584364435979, + "kl": 0.031982421875, + "learning_rate": 1.0177514792899408e-07, + "loss": 0.0013, + "reward": 0.171875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1518 + }, + { + "completion_length": 114.7890625, + "epoch": 0.8988165680473372, + "grad_norm": 0.7517597762701701, + "kl": 0.031494140625, + "learning_rate": 1.0118343195266271e-07, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1519 + }, + { + "completion_length": 119.2734375, + "epoch": 0.8994082840236687, + "grad_norm": 0.4590912369946038, + "kl": 0.0189208984375, + "learning_rate": 1.0059171597633135e-07, + "loss": 0.0008, + "reward": 0.1171875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1520 + }, + { + "completion_length": 105.703125, + "epoch": 0.9, + "grad_norm": 0.6165603120098826, + "kl": 0.033203125, + "learning_rate": 1e-07, + "loss": 0.0013, + "reward": 0.234375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.234375, + "step": 1521 + }, + { + "completion_length": 118.671875, + "epoch": 0.9005917159763314, + "grad_norm": 0.27009583008161364, + "kl": 0.0303955078125, + "learning_rate": 9.940828402366863e-08, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1522 + }, + { + "completion_length": 119.5859375, + "epoch": 0.9011834319526627, + "grad_norm": 0.860680407395948, + "kl": 0.033203125, + "learning_rate": 9.881656804733728e-08, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1523 + }, + { + "completion_length": 112.7109375, + "epoch": 0.901775147928994, + "grad_norm": 0.6638872499198977, + "kl": 0.026123046875, + "learning_rate": 9.822485207100591e-08, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1524 + }, + { + "completion_length": 115.0, + "epoch": 0.9023668639053254, + "grad_norm": 0.35511004969746185, + "kl": 0.033935546875, + "learning_rate": 9.763313609467456e-08, + "loss": 0.0014, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1525 + }, + { + "completion_length": 121.9375, + "epoch": 0.9029585798816568, + "grad_norm": 1.1481196984045359, + "kl": 0.023681640625, + "learning_rate": 9.70414201183432e-08, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1526 + }, + { + "completion_length": 120.1484375, + "epoch": 0.9035502958579882, + "grad_norm": 0.47624637655356994, + "kl": 0.03369140625, + "learning_rate": 9.644970414201183e-08, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1527 + }, + { + "completion_length": 118.671875, + "epoch": 0.9041420118343195, + "grad_norm": 0.9146926263236542, + "kl": 0.033447265625, + "learning_rate": 9.585798816568047e-08, + "loss": 0.0013, + "reward": 0.1796875, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 1528 + }, + { + "completion_length": 119.703125, + "epoch": 0.9047337278106509, + "grad_norm": 0.2981832534895869, + "kl": 0.027587890625, + "learning_rate": 9.526627218934911e-08, + "loss": 0.0011, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1529 + }, + { + "completion_length": 118.4921875, + "epoch": 0.9053254437869822, + "grad_norm": 1.054195565071822, + "kl": 0.0380859375, + "learning_rate": 9.467455621301774e-08, + "loss": 0.0015, + "reward": 0.109375, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1530 + }, + { + "completion_length": 116.046875, + "epoch": 0.9059171597633137, + "grad_norm": 0.9287178615131211, + "kl": 0.03564453125, + "learning_rate": 9.408284023668639e-08, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1531 + }, + { + "completion_length": 124.9453125, + "epoch": 0.906508875739645, + "grad_norm": 0.6299876469817517, + "kl": 0.041015625, + "learning_rate": 9.349112426035502e-08, + "loss": 0.0016, + "reward": 0.15625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1532 + }, + { + "completion_length": 127.75, + "epoch": 0.9071005917159763, + "grad_norm": 0.42453529296251463, + "kl": 0.030029296875, + "learning_rate": 9.289940828402366e-08, + "loss": 0.0012, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1533 + }, + { + "completion_length": 109.140625, + "epoch": 0.9076923076923077, + "grad_norm": 1.5930822445300177, + "kl": 0.03125, + "learning_rate": 9.23076923076923e-08, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.11420938372612, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1534 + }, + { + "completion_length": 118.859375, + "epoch": 0.908284023668639, + "grad_norm": 2.0217725834426505, + "kl": 0.0263671875, + "learning_rate": 9.171597633136093e-08, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1535 + }, + { + "completion_length": 122.9140625, + "epoch": 0.9088757396449704, + "grad_norm": 0.4035145955641133, + "kl": 0.0157470703125, + "learning_rate": 9.112426035502958e-08, + "loss": 0.0006, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1536 + }, + { + "completion_length": 134.2265625, + "epoch": 0.9094674556213018, + "grad_norm": 0.4172483910396547, + "kl": 0.0225830078125, + "learning_rate": 9.053254437869821e-08, + "loss": 0.0009, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1537 + }, + { + "completion_length": 125.2578125, + "epoch": 0.9100591715976332, + "grad_norm": 0.9679345578008811, + "kl": 0.0262451171875, + "learning_rate": 8.994082840236686e-08, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1538 + }, + { + "completion_length": 125.1171875, + "epoch": 0.9106508875739645, + "grad_norm": 0.4036879370790311, + "kl": 0.0216064453125, + "learning_rate": 8.934911242603551e-08, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1539 + }, + { + "completion_length": 124.9296875, + "epoch": 0.9112426035502958, + "grad_norm": 0.39247998813679813, + "kl": 0.0257568359375, + "learning_rate": 8.875739644970414e-08, + "loss": 0.001, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1540 + }, + { + "completion_length": 115.6015625, + "epoch": 0.9118343195266272, + "grad_norm": 0.42952745716748914, + "kl": 0.025634765625, + "learning_rate": 8.816568047337278e-08, + "loss": 0.001, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1541 + }, + { + "completion_length": 122.0625, + "epoch": 0.9124260355029585, + "grad_norm": 0.7338995907497657, + "kl": 0.037841796875, + "learning_rate": 8.757396449704142e-08, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1542 + }, + { + "completion_length": 123.515625, + "epoch": 0.91301775147929, + "grad_norm": 0.4504356821885895, + "kl": 0.043212890625, + "learning_rate": 8.698224852071005e-08, + "loss": 0.0017, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1543 + }, + { + "completion_length": 119.5703125, + "epoch": 0.9136094674556213, + "grad_norm": 0.7932326351626987, + "kl": 0.037841796875, + "learning_rate": 8.63905325443787e-08, + "loss": 0.0015, + "reward": 0.21875, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.21875, + "step": 1544 + }, + { + "completion_length": 111.6875, + "epoch": 0.9142011834319527, + "grad_norm": 0.5047247784036191, + "kl": 0.035400390625, + "learning_rate": 8.579881656804733e-08, + "loss": 0.0014, + "reward": 0.1796875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1796875, + "step": 1545 + }, + { + "completion_length": 114.875, + "epoch": 0.914792899408284, + "grad_norm": 0.45194314805085234, + "kl": 0.02783203125, + "learning_rate": 8.520710059171597e-08, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1546 + }, + { + "completion_length": 124.5703125, + "epoch": 0.9153846153846154, + "grad_norm": 0.7153372455546799, + "kl": 0.036865234375, + "learning_rate": 8.461538461538461e-08, + "loss": 0.0015, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1547 + }, + { + "completion_length": 124.6328125, + "epoch": 0.9159763313609467, + "grad_norm": 0.39200712765232726, + "kl": 0.0361328125, + "learning_rate": 8.402366863905324e-08, + "loss": 0.0014, + "reward": 0.109375, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1548 + }, + { + "completion_length": 117.7890625, + "epoch": 0.9165680473372781, + "grad_norm": 0.4326004290526298, + "kl": 0.03271484375, + "learning_rate": 8.343195266272189e-08, + "loss": 0.0013, + "reward": 0.0234375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1549 + }, + { + "completion_length": 130.75, + "epoch": 0.9171597633136095, + "grad_norm": 0.5207465885147188, + "kl": 0.027099609375, + "learning_rate": 8.284023668639053e-08, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1550 + }, + { + "completion_length": 124.2265625, + "epoch": 0.9177514792899408, + "grad_norm": 1.799655302883567, + "kl": 0.0322265625, + "learning_rate": 8.224852071005916e-08, + "loss": 0.0013, + "reward": 0.125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1551 + }, + { + "completion_length": 123.3984375, + "epoch": 0.9183431952662722, + "grad_norm": 0.6821427838946937, + "kl": 0.02880859375, + "learning_rate": 8.165680473372782e-08, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1552 + }, + { + "completion_length": 122.46875, + "epoch": 0.9189349112426035, + "grad_norm": 0.06103025069813044, + "kl": 0.0244140625, + "learning_rate": 8.106508875739645e-08, + "loss": 0.001, + "reward": 0.15625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1553 + }, + { + "completion_length": 139.5390625, + "epoch": 0.919526627218935, + "grad_norm": 0.3204921499529041, + "kl": 0.025146484375, + "learning_rate": 8.047337278106509e-08, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1554 + }, + { + "completion_length": 123.203125, + "epoch": 0.9201183431952663, + "grad_norm": 0.608060247363348, + "kl": 0.030517578125, + "learning_rate": 7.988165680473373e-08, + "loss": 0.0012, + "reward": 0.171875, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1555 + }, + { + "completion_length": 115.21875, + "epoch": 0.9207100591715977, + "grad_norm": 0.37463617668077964, + "kl": 0.02197265625, + "learning_rate": 7.928994082840236e-08, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1556 + }, + { + "completion_length": 119.9453125, + "epoch": 0.921301775147929, + "grad_norm": 0.40503113516799905, + "kl": 0.033203125, + "learning_rate": 7.8698224852071e-08, + "loss": 0.0013, + "reward": 0.109375, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1557 + }, + { + "completion_length": 117.9140625, + "epoch": 0.9218934911242603, + "grad_norm": 0.45815582961813295, + "kl": 0.03173828125, + "learning_rate": 7.810650887573964e-08, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1558 + }, + { + "completion_length": 118.0, + "epoch": 0.9224852071005917, + "grad_norm": 0.37710388824306423, + "kl": 0.031982421875, + "learning_rate": 7.751479289940828e-08, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1559 + }, + { + "completion_length": 113.8515625, + "epoch": 0.9230769230769231, + "grad_norm": 0.6998831937717842, + "kl": 0.03515625, + "learning_rate": 7.692307692307692e-08, + "loss": 0.0014, + "reward": 0.1171875, + "reward_std": 0.10100159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1560 + }, + { + "completion_length": 127.5703125, + "epoch": 0.9236686390532545, + "grad_norm": 1.0256058137933692, + "kl": 0.0198974609375, + "learning_rate": 7.633136094674555e-08, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1561 + }, + { + "completion_length": 123.6875, + "epoch": 0.9242603550295858, + "grad_norm": 0.563120780383989, + "kl": 0.02783203125, + "learning_rate": 7.57396449704142e-08, + "loss": 0.0011, + "reward": 0.109375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1562 + }, + { + "completion_length": 128.9140625, + "epoch": 0.9248520710059172, + "grad_norm": 0.5227757518492521, + "kl": 0.029541015625, + "learning_rate": 7.514792899408284e-08, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1563 + }, + { + "completion_length": 121.8203125, + "epoch": 0.9254437869822485, + "grad_norm": 0.4159321863061787, + "kl": 0.0257568359375, + "learning_rate": 7.455621301775147e-08, + "loss": 0.001, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1564 + }, + { + "completion_length": 128.6953125, + "epoch": 0.9260355029585798, + "grad_norm": 0.2597894418093026, + "kl": 0.0233154296875, + "learning_rate": 7.396449704142011e-08, + "loss": 0.0009, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 1565 + }, + { + "completion_length": 124.0078125, + "epoch": 0.9266272189349113, + "grad_norm": 1.9895209618276088, + "kl": 0.0179443359375, + "learning_rate": 7.337278106508874e-08, + "loss": 0.0007, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1566 + }, + { + "completion_length": 116.3359375, + "epoch": 0.9272189349112426, + "grad_norm": 0.5245534854931906, + "kl": 0.017578125, + "learning_rate": 7.27810650887574e-08, + "loss": 0.0007, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1567 + }, + { + "completion_length": 126.9453125, + "epoch": 0.927810650887574, + "grad_norm": 0.7128410798989901, + "kl": 0.023681640625, + "learning_rate": 7.218934911242604e-08, + "loss": 0.0009, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1568 + }, + { + "completion_length": 120.515625, + "epoch": 0.9284023668639053, + "grad_norm": 2.5755660474043274, + "kl": 0.0272216796875, + "learning_rate": 7.159763313609467e-08, + "loss": 0.0011, + "reward": 0.1484375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1569 + }, + { + "completion_length": 120.8046875, + "epoch": 0.9289940828402367, + "grad_norm": 0.45544808583558455, + "kl": 0.0299072265625, + "learning_rate": 7.100591715976332e-08, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1570 + }, + { + "completion_length": 132.359375, + "epoch": 0.929585798816568, + "grad_norm": 0.26872274905639215, + "kl": 0.02587890625, + "learning_rate": 7.041420118343195e-08, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1571 + }, + { + "completion_length": 121.859375, + "epoch": 0.9301775147928995, + "grad_norm": 6.817015084150098, + "kl": 0.028076171875, + "learning_rate": 6.982248520710059e-08, + "loss": 0.0011, + "reward": 0.15625, + "reward_std": 0.09375, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1572 + }, + { + "completion_length": 120.6171875, + "epoch": 0.9307692307692308, + "grad_norm": 0.6196824814004842, + "kl": 0.03173828125, + "learning_rate": 6.923076923076923e-08, + "loss": 0.0013, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1573 + }, + { + "completion_length": 129.140625, + "epoch": 0.9313609467455621, + "grad_norm": 0.3352584227850343, + "kl": 0.022705078125, + "learning_rate": 6.863905325443786e-08, + "loss": 0.0009, + "reward": 0.1484375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1574 + }, + { + "completion_length": 113.9296875, + "epoch": 0.9319526627218935, + "grad_norm": 0.6425515909953766, + "kl": 0.03125, + "learning_rate": 6.80473372781065e-08, + "loss": 0.0012, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1575 + }, + { + "completion_length": 126.6171875, + "epoch": 0.9325443786982248, + "grad_norm": 0.48963176733144065, + "kl": 0.0244140625, + "learning_rate": 6.745562130177515e-08, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1576 + }, + { + "completion_length": 126.421875, + "epoch": 0.9331360946745562, + "grad_norm": 0.5702596808923401, + "kl": 0.0240478515625, + "learning_rate": 6.686390532544378e-08, + "loss": 0.001, + "reward": 0.125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1577 + }, + { + "completion_length": 121.8828125, + "epoch": 0.9337278106508876, + "grad_norm": 0.32999306283527435, + "kl": 0.0255126953125, + "learning_rate": 6.627218934911242e-08, + "loss": 0.001, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1578 + }, + { + "completion_length": 124.109375, + "epoch": 0.934319526627219, + "grad_norm": 0.442498182113782, + "kl": 0.023681640625, + "learning_rate": 6.568047337278105e-08, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1579 + }, + { + "completion_length": 127.171875, + "epoch": 0.9349112426035503, + "grad_norm": 1.6177213716740915, + "kl": 0.03369140625, + "learning_rate": 6.50887573964497e-08, + "loss": 0.0013, + "reward": 0.09375, + "reward_std": 0.07216878235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1580 + }, + { + "completion_length": 114.7265625, + "epoch": 0.9355029585798816, + "grad_norm": 0.824091222092233, + "kl": 0.03125, + "learning_rate": 6.449704142011835e-08, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1581 + }, + { + "completion_length": 129.28125, + "epoch": 0.936094674556213, + "grad_norm": 1.1219659709104721, + "kl": 0.031005859375, + "learning_rate": 6.390532544378698e-08, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1582 + }, + { + "completion_length": 118.25, + "epoch": 0.9366863905325444, + "grad_norm": 0.6091740496549829, + "kl": 0.0205078125, + "learning_rate": 6.331360946745562e-08, + "loss": 0.0008, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1583 + }, + { + "completion_length": 118.9921875, + "epoch": 0.9372781065088758, + "grad_norm": 0.7246864856590567, + "kl": 0.035400390625, + "learning_rate": 6.272189349112427e-08, + "loss": 0.0014, + "reward": 0.140625, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1584 + }, + { + "completion_length": 119.8046875, + "epoch": 0.9378698224852071, + "grad_norm": 0.5475343076179838, + "kl": 0.03173828125, + "learning_rate": 6.21301775147929e-08, + "loss": 0.0013, + "reward": 0.078125, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1585 + }, + { + "completion_length": 120.7109375, + "epoch": 0.9384615384615385, + "grad_norm": 0.7364080836904903, + "kl": 0.035888671875, + "learning_rate": 6.153846153846154e-08, + "loss": 0.0014, + "reward": 0.1328125, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1586 + }, + { + "completion_length": 118.5390625, + "epoch": 0.9390532544378698, + "grad_norm": 0.2768616385938826, + "kl": 0.02587890625, + "learning_rate": 6.094674556213017e-08, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1587 + }, + { + "completion_length": 131.25, + "epoch": 0.9396449704142011, + "grad_norm": 0.4890542580903742, + "kl": 0.03173828125, + "learning_rate": 6.035502958579881e-08, + "loss": 0.0013, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1588 + }, + { + "completion_length": 118.75, + "epoch": 0.9402366863905326, + "grad_norm": 0.5237220044218782, + "kl": 0.025634765625, + "learning_rate": 5.976331360946746e-08, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1589 + }, + { + "completion_length": 130.5625, + "epoch": 0.9408284023668639, + "grad_norm": 0.4265125014003028, + "kl": 0.0238037109375, + "learning_rate": 5.917159763313609e-08, + "loss": 0.0009, + "reward": 0.03125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1590 + }, + { + "completion_length": 119.453125, + "epoch": 0.9414201183431953, + "grad_norm": 0.0642270528865133, + "kl": 0.02392578125, + "learning_rate": 5.857988165680473e-08, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1591 + }, + { + "completion_length": 128.5390625, + "epoch": 0.9420118343195266, + "grad_norm": 1.0279895133994081, + "kl": 0.03662109375, + "learning_rate": 5.7988165680473374e-08, + "loss": 0.0015, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1592 + }, + { + "completion_length": 114.8828125, + "epoch": 0.942603550295858, + "grad_norm": 0.6226704382247332, + "kl": 0.0286865234375, + "learning_rate": 5.739644970414201e-08, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1593 + }, + { + "completion_length": 130.3828125, + "epoch": 0.9431952662721893, + "grad_norm": 0.5903373648610195, + "kl": 0.0269775390625, + "learning_rate": 5.680473372781065e-08, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.06733439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1594 + }, + { + "completion_length": 118.3515625, + "epoch": 0.9437869822485208, + "grad_norm": 0.28629175494874, + "kl": 0.0157470703125, + "learning_rate": 5.621301775147929e-08, + "loss": 0.0006, + "reward": 0.0703125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1595 + }, + { + "completion_length": 115.40625, + "epoch": 0.9443786982248521, + "grad_norm": 0.5660573583196825, + "kl": 0.032958984375, + "learning_rate": 5.562130177514793e-08, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1596 + }, + { + "completion_length": 127.84375, + "epoch": 0.9449704142011834, + "grad_norm": 0.5449391762990698, + "kl": 0.0225830078125, + "learning_rate": 5.5029585798816563e-08, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1597 + }, + { + "completion_length": 126.3984375, + "epoch": 0.9455621301775148, + "grad_norm": 0.5275088137563274, + "kl": 0.033447265625, + "learning_rate": 5.44378698224852e-08, + "loss": 0.0013, + "reward": 0.0703125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1598 + }, + { + "completion_length": 128.4140625, + "epoch": 0.9461538461538461, + "grad_norm": 0.5161993322242041, + "kl": 0.0306396484375, + "learning_rate": 5.384615384615385e-08, + "loss": 0.0012, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1599 + }, + { + "completion_length": 117.046875, + "epoch": 0.9467455621301775, + "grad_norm": 0.5556252319556066, + "kl": 0.02783203125, + "learning_rate": 5.3254437869822486e-08, + "loss": 0.0011, + "reward": 0.0625, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1600 + }, + { + "completion_length": 120.7421875, + "epoch": 0.9473372781065089, + "grad_norm": 0.9864054617871734, + "kl": 0.045166015625, + "learning_rate": 5.266272189349112e-08, + "loss": 0.0018, + "reward": 0.1328125, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1601 + }, + { + "completion_length": 128.421875, + "epoch": 0.9479289940828403, + "grad_norm": 0.331076102165219, + "kl": 0.0225830078125, + "learning_rate": 5.207100591715976e-08, + "loss": 0.0009, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1602 + }, + { + "completion_length": 120.140625, + "epoch": 0.9485207100591716, + "grad_norm": 0.06994695920805027, + "kl": 0.0260009765625, + "learning_rate": 5.14792899408284e-08, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1603 + }, + { + "completion_length": 122.6875, + "epoch": 0.9491124260355029, + "grad_norm": 0.6250884021438864, + "kl": 0.031982421875, + "learning_rate": 5.088757396449704e-08, + "loss": 0.0013, + "reward": 0.1328125, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1604 + }, + { + "completion_length": 112.828125, + "epoch": 0.9497041420118343, + "grad_norm": 0.4963075423794173, + "kl": 0.033447265625, + "learning_rate": 5.0295857988165676e-08, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1605 + }, + { + "completion_length": 117.9140625, + "epoch": 0.9502958579881656, + "grad_norm": 0.6067398175662789, + "kl": 0.029052734375, + "learning_rate": 4.970414201183431e-08, + "loss": 0.0012, + "reward": 0.1875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1875, + "step": 1606 + }, + { + "completion_length": 126.34375, + "epoch": 0.9508875739644971, + "grad_norm": 0.5742654991521986, + "kl": 0.024658203125, + "learning_rate": 4.9112426035502956e-08, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1607 + }, + { + "completion_length": 118.8984375, + "epoch": 0.9514792899408284, + "grad_norm": 0.8505311649720412, + "kl": 0.0361328125, + "learning_rate": 4.85207100591716e-08, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1608 + }, + { + "completion_length": 126.359375, + "epoch": 0.9520710059171598, + "grad_norm": 0.5968507721853066, + "kl": 0.02978515625, + "learning_rate": 4.7928994082840236e-08, + "loss": 0.0012, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1609 + }, + { + "completion_length": 115.796875, + "epoch": 0.9526627218934911, + "grad_norm": 0.06494535212880768, + "kl": 0.03515625, + "learning_rate": 4.733727810650887e-08, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1610 + }, + { + "completion_length": 120.2578125, + "epoch": 0.9532544378698224, + "grad_norm": 0.5474682787879456, + "kl": 0.0235595703125, + "learning_rate": 4.674556213017751e-08, + "loss": 0.0009, + "reward": 0.078125, + "reward_std": 0.05412658676505089, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1611 + }, + { + "completion_length": 122.6953125, + "epoch": 0.9538461538461539, + "grad_norm": 0.7892396383437278, + "kl": 0.0306396484375, + "learning_rate": 4.615384615384615e-08, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1612 + }, + { + "completion_length": 123.1640625, + "epoch": 0.9544378698224852, + "grad_norm": 0.6091599356569696, + "kl": 0.021240234375, + "learning_rate": 4.556213017751479e-08, + "loss": 0.0008, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1613 + }, + { + "completion_length": 126.25, + "epoch": 0.9550295857988166, + "grad_norm": 0.2668777900071238, + "kl": 0.019775390625, + "learning_rate": 4.497041420118343e-08, + "loss": 0.0008, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1614 + }, + { + "completion_length": 114.859375, + "epoch": 0.9556213017751479, + "grad_norm": 0.5112255211124955, + "kl": 0.0341796875, + "learning_rate": 4.437869822485207e-08, + "loss": 0.0014, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1615 + }, + { + "completion_length": 114.8515625, + "epoch": 0.9562130177514793, + "grad_norm": 1.0753631831445212, + "kl": 0.0286865234375, + "learning_rate": 4.378698224852071e-08, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1616 + }, + { + "completion_length": 113.5, + "epoch": 0.9568047337278106, + "grad_norm": 0.4616437664658907, + "kl": 0.0341796875, + "learning_rate": 4.319526627218935e-08, + "loss": 0.0014, + "reward": 0.09375, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1617 + }, + { + "completion_length": 128.7265625, + "epoch": 0.9573964497041421, + "grad_norm": 0.5994034032575735, + "kl": 0.02587890625, + "learning_rate": 4.2603550295857985e-08, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1618 + }, + { + "completion_length": 122.953125, + "epoch": 0.9579881656804734, + "grad_norm": 0.27743569821208874, + "kl": 0.0245361328125, + "learning_rate": 4.201183431952662e-08, + "loss": 0.001, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1619 + }, + { + "completion_length": 122.7890625, + "epoch": 0.9585798816568047, + "grad_norm": 0.38272551031046165, + "kl": 0.032470703125, + "learning_rate": 4.1420118343195265e-08, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1620 + }, + { + "completion_length": 128.046875, + "epoch": 0.9591715976331361, + "grad_norm": 0.7362560311263822, + "kl": 0.04345703125, + "learning_rate": 4.082840236686391e-08, + "loss": 0.0017, + "reward": 0.1328125, + "reward_std": 0.08295939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1621 + }, + { + "completion_length": 128.9765625, + "epoch": 0.9597633136094674, + "grad_norm": 0.603255659921747, + "kl": 0.029541015625, + "learning_rate": 4.0236686390532545e-08, + "loss": 0.0012, + "reward": 0.1015625, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1622 + }, + { + "completion_length": 129.5390625, + "epoch": 0.9603550295857988, + "grad_norm": 0.500327863187724, + "kl": 0.0277099609375, + "learning_rate": 3.964497041420118e-08, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.03608439117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1623 + }, + { + "completion_length": 132.1640625, + "epoch": 0.9609467455621302, + "grad_norm": 3.253159674290781, + "kl": 0.04150390625, + "learning_rate": 3.905325443786982e-08, + "loss": 0.0017, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1624 + }, + { + "completion_length": 113.4921875, + "epoch": 0.9615384615384616, + "grad_norm": 0.4911014426363469, + "kl": 0.02734375, + "learning_rate": 3.846153846153846e-08, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1625 + }, + { + "completion_length": 126.9921875, + "epoch": 0.9621301775147929, + "grad_norm": 1.7774409175642365, + "kl": 0.0308837890625, + "learning_rate": 3.78698224852071e-08, + "loss": 0.0012, + "reward": 0.109375, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.109375, + "step": 1626 + }, + { + "completion_length": 118.6640625, + "epoch": 0.9627218934911242, + "grad_norm": 0.47590246649649354, + "kl": 0.0291748046875, + "learning_rate": 3.7278106508875735e-08, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1627 + }, + { + "completion_length": 123.4140625, + "epoch": 0.9633136094674556, + "grad_norm": 0.5807596296150406, + "kl": 0.0301513671875, + "learning_rate": 3.668639053254437e-08, + "loss": 0.0012, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1628 + }, + { + "completion_length": 127.7734375, + "epoch": 0.9639053254437869, + "grad_norm": 0.7056687648566846, + "kl": 0.0213623046875, + "learning_rate": 3.609467455621302e-08, + "loss": 0.0009, + "reward": 0.015625, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.015625, + "step": 1629 + }, + { + "completion_length": 121.7578125, + "epoch": 0.9644970414201184, + "grad_norm": 0.3445927163221434, + "kl": 0.0216064453125, + "learning_rate": 3.550295857988166e-08, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1630 + }, + { + "completion_length": 116.796875, + "epoch": 0.9650887573964497, + "grad_norm": 0.4248506405353525, + "kl": 0.025146484375, + "learning_rate": 3.4911242603550294e-08, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1631 + }, + { + "completion_length": 132.1875, + "epoch": 0.9656804733727811, + "grad_norm": 0.043321584477602905, + "kl": 0.0205078125, + "learning_rate": 3.431952662721893e-08, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1632 + }, + { + "completion_length": 131.765625, + "epoch": 0.9662721893491124, + "grad_norm": 0.5811415211095164, + "kl": 0.0245361328125, + "learning_rate": 3.3727810650887574e-08, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1633 + }, + { + "completion_length": 131.6171875, + "epoch": 0.9668639053254438, + "grad_norm": 0.6426404729878841, + "kl": 0.0264892578125, + "learning_rate": 3.313609467455621e-08, + "loss": 0.0011, + "reward": 0.125, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1634 + }, + { + "completion_length": 124.578125, + "epoch": 0.9674556213017751, + "grad_norm": 0.7128613739419764, + "kl": 0.0263671875, + "learning_rate": 3.254437869822485e-08, + "loss": 0.0011, + "reward": 0.078125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1635 + }, + { + "completion_length": 126.59375, + "epoch": 0.9680473372781065, + "grad_norm": 0.4878695363991926, + "kl": 0.024658203125, + "learning_rate": 3.195266272189349e-08, + "loss": 0.001, + "reward": 0.078125, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.078125, + "step": 1636 + }, + { + "completion_length": 128.8515625, + "epoch": 0.9686390532544379, + "grad_norm": 0.40270295579179693, + "kl": 0.03369140625, + "learning_rate": 3.1360946745562134e-08, + "loss": 0.0013, + "reward": 0.140625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1637 + }, + { + "completion_length": 128.21875, + "epoch": 0.9692307692307692, + "grad_norm": 0.620213470631193, + "kl": 0.0263671875, + "learning_rate": 3.076923076923077e-08, + "loss": 0.0011, + "reward": 0.046875, + "reward_std": 0.0625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1638 + }, + { + "completion_length": 131.640625, + "epoch": 0.9698224852071006, + "grad_norm": 0.25290589472525543, + "kl": 0.0294189453125, + "learning_rate": 3.017751479289941e-08, + "loss": 0.0012, + "reward": 0.0078125, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0078125, + "step": 1639 + }, + { + "completion_length": 126.921875, + "epoch": 0.9704142011834319, + "grad_norm": 0.400560524609336, + "kl": 0.03271484375, + "learning_rate": 2.9585798816568044e-08, + "loss": 0.0013, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1640 + }, + { + "completion_length": 119.9375, + "epoch": 0.9710059171597634, + "grad_norm": 0.9567437233987502, + "kl": 0.033203125, + "learning_rate": 2.8994082840236687e-08, + "loss": 0.0013, + "reward": 0.15625, + "reward_std": 0.125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1641 + }, + { + "completion_length": 118.0859375, + "epoch": 0.9715976331360947, + "grad_norm": 0.3366152670411402, + "kl": 0.0269775390625, + "learning_rate": 2.8402366863905323e-08, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1642 + }, + { + "completion_length": 133.8046875, + "epoch": 0.972189349112426, + "grad_norm": 0.48886963976845843, + "kl": 0.0233154296875, + "learning_rate": 2.7810650887573963e-08, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1643 + }, + { + "completion_length": 121.7109375, + "epoch": 0.9727810650887574, + "grad_norm": 0.4119256294223336, + "kl": 0.021484375, + "learning_rate": 2.72189349112426e-08, + "loss": 0.0009, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1644 + }, + { + "completion_length": 129.0390625, + "epoch": 0.9733727810650887, + "grad_norm": 0.1217263320514882, + "kl": 0.028076171875, + "learning_rate": 2.6627218934911243e-08, + "loss": 0.0011, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1645 + }, + { + "completion_length": 126.765625, + "epoch": 0.9739644970414201, + "grad_norm": 0.8241528728211804, + "kl": 0.0294189453125, + "learning_rate": 2.603550295857988e-08, + "loss": 0.0012, + "reward": 0.140625, + "reward_std": 0.08537659049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.140625, + "step": 1646 + }, + { + "completion_length": 131.5390625, + "epoch": 0.9745562130177515, + "grad_norm": 0.04902347577601507, + "kl": 0.0157470703125, + "learning_rate": 2.544378698224852e-08, + "loss": 0.0006, + "reward": 0.09375, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.09375, + "step": 1647 + }, + { + "completion_length": 119.4140625, + "epoch": 0.9751479289940829, + "grad_norm": 0.556742694767248, + "kl": 0.03173828125, + "learning_rate": 2.4852071005917156e-08, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.06975159049034119, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1648 + }, + { + "completion_length": 121.9296875, + "epoch": 0.9757396449704142, + "grad_norm": 0.5078088650483421, + "kl": 0.035888671875, + "learning_rate": 2.42603550295858e-08, + "loss": 0.0014, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1649 + }, + { + "completion_length": 125.484375, + "epoch": 0.9763313609467456, + "grad_norm": 0.3830337001821043, + "kl": 0.019775390625, + "learning_rate": 2.3668639053254436e-08, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1650 + }, + { + "completion_length": 124.5390625, + "epoch": 0.9769230769230769, + "grad_norm": 0.7040653970025333, + "kl": 0.03662109375, + "learning_rate": 2.3076923076923076e-08, + "loss": 0.0015, + "reward": 0.1171875, + "reward_std": 0.08779378235340118, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1651 + }, + { + "completion_length": 117.7734375, + "epoch": 0.9775147928994082, + "grad_norm": 0.3299598734743406, + "kl": 0.022216796875, + "learning_rate": 2.2485207100591716e-08, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1652 + }, + { + "completion_length": 123.328125, + "epoch": 0.9781065088757397, + "grad_norm": 0.41763116459498006, + "kl": 0.03173828125, + "learning_rate": 2.1893491124260356e-08, + "loss": 0.0013, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1653 + }, + { + "completion_length": 121.6484375, + "epoch": 0.978698224852071, + "grad_norm": 1.059463338319333, + "kl": 0.02880859375, + "learning_rate": 2.1301775147928993e-08, + "loss": 0.0012, + "reward": 0.1640625, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1640625, + "step": 1654 + }, + { + "completion_length": 119.5390625, + "epoch": 0.9792899408284024, + "grad_norm": 0.5122973216511356, + "kl": 0.0308837890625, + "learning_rate": 2.0710059171597633e-08, + "loss": 0.0012, + "reward": 0.1328125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1655 + }, + { + "completion_length": 120.4921875, + "epoch": 0.9798816568047337, + "grad_norm": 0.4019742965349983, + "kl": 0.04443359375, + "learning_rate": 2.0118343195266272e-08, + "loss": 0.0018, + "reward": 0.1328125, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1328125, + "step": 1656 + }, + { + "completion_length": 123.59375, + "epoch": 0.9804733727810651, + "grad_norm": 0.8060045690301411, + "kl": 0.034912109375, + "learning_rate": 1.952662721893491e-08, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1657 + }, + { + "completion_length": 120.8359375, + "epoch": 0.9810650887573964, + "grad_norm": 0.5851928557578986, + "kl": 0.032470703125, + "learning_rate": 1.893491124260355e-08, + "loss": 0.0013, + "reward": 0.0859375, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1658 + }, + { + "completion_length": 125.2421875, + "epoch": 0.9816568047337279, + "grad_norm": 0.6150399975320567, + "kl": 0.0260009765625, + "learning_rate": 1.8343195266272186e-08, + "loss": 0.001, + "reward": 0.0546875, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1659 + }, + { + "completion_length": 123.90625, + "epoch": 0.9822485207100592, + "grad_norm": 0.47729563086187643, + "kl": 0.031494140625, + "learning_rate": 1.775147928994083e-08, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1660 + }, + { + "completion_length": 126.703125, + "epoch": 0.9828402366863905, + "grad_norm": 0.631103593024437, + "kl": 0.0308837890625, + "learning_rate": 1.7159763313609465e-08, + "loss": 0.0012, + "reward": 0.1484375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1661 + }, + { + "completion_length": 133.3671875, + "epoch": 0.9834319526627219, + "grad_norm": 0.8524143393550843, + "kl": 0.027099609375, + "learning_rate": 1.6568047337278105e-08, + "loss": 0.0011, + "reward": 0.0546875, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1662 + }, + { + "completion_length": 123.484375, + "epoch": 0.9840236686390532, + "grad_norm": 0.28880502123351703, + "kl": 0.02392578125, + "learning_rate": 1.5976331360946745e-08, + "loss": 0.001, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1663 + }, + { + "completion_length": 130.328125, + "epoch": 0.9846153846153847, + "grad_norm": 0.4131727948195972, + "kl": 0.02587890625, + "learning_rate": 1.5384615384615385e-08, + "loss": 0.001, + "reward": 0.046875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1664 + }, + { + "completion_length": 135.71875, + "epoch": 0.985207100591716, + "grad_norm": 0.5470978764638946, + "kl": 0.022216796875, + "learning_rate": 1.4792899408284022e-08, + "loss": 0.0009, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1665 + }, + { + "completion_length": 125.7421875, + "epoch": 0.9857988165680474, + "grad_norm": 0.5190509879907931, + "kl": 0.02490234375, + "learning_rate": 1.4201183431952662e-08, + "loss": 0.001, + "reward": 0.0625, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1666 + }, + { + "completion_length": 121.921875, + "epoch": 0.9863905325443787, + "grad_norm": 0.37132575946965707, + "kl": 0.0159912109375, + "learning_rate": 1.36094674556213e-08, + "loss": 0.0006, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1667 + }, + { + "completion_length": 122.6796875, + "epoch": 0.98698224852071, + "grad_norm": 1.0337427055053692, + "kl": 0.03466796875, + "learning_rate": 1.301775147928994e-08, + "loss": 0.0014, + "reward": 0.125, + "reward_std": 0.08054219186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.125, + "step": 1668 + }, + { + "completion_length": 122.5234375, + "epoch": 0.9875739644970414, + "grad_norm": 0.49649716934129823, + "kl": 0.0203857421875, + "learning_rate": 1.2426035502958578e-08, + "loss": 0.0008, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1669 + }, + { + "completion_length": 121.6171875, + "epoch": 0.9881656804733728, + "grad_norm": 0.8885089670764802, + "kl": 0.0250244140625, + "learning_rate": 1.1834319526627218e-08, + "loss": 0.001, + "reward": 0.0390625, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1670 + }, + { + "completion_length": 119.0078125, + "epoch": 0.9887573964497042, + "grad_norm": 0.5468495527683733, + "kl": 0.034912109375, + "learning_rate": 1.1242603550295858e-08, + "loss": 0.0014, + "reward": 0.1484375, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1671 + }, + { + "completion_length": 120.3046875, + "epoch": 0.9893491124260355, + "grad_norm": 0.6223568140446142, + "kl": 0.03369140625, + "learning_rate": 1.0650887573964496e-08, + "loss": 0.0013, + "reward": 0.15625, + "reward_std": 0.049292195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.15625, + "step": 1672 + }, + { + "completion_length": 121.671875, + "epoch": 0.9899408284023669, + "grad_norm": 0.2987970725773147, + "kl": 0.023681640625, + "learning_rate": 1.0059171597633136e-08, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1673 + }, + { + "completion_length": 120.5859375, + "epoch": 0.9905325443786982, + "grad_norm": 0.6903168208692936, + "kl": 0.03173828125, + "learning_rate": 9.467455621301774e-09, + "loss": 0.0013, + "reward": 0.1015625, + "reward_std": 0.06491719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1674 + }, + { + "completion_length": 124.9921875, + "epoch": 0.9911242603550295, + "grad_norm": 0.04628854629662004, + "kl": 0.017822265625, + "learning_rate": 8.875739644970414e-09, + "loss": 0.0007, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1675 + }, + { + "completion_length": 117.7734375, + "epoch": 0.991715976331361, + "grad_norm": 0.55105753578662, + "kl": 0.0216064453125, + "learning_rate": 8.284023668639053e-09, + "loss": 0.0009, + "reward": 0.0546875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0546875, + "step": 1676 + }, + { + "completion_length": 113.4375, + "epoch": 0.9923076923076923, + "grad_norm": 0.43891714038022284, + "kl": 0.025390625, + "learning_rate": 7.692307692307693e-09, + "loss": 0.001, + "reward": 0.03125, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.03125, + "step": 1677 + }, + { + "completion_length": 114.5546875, + "epoch": 0.9928994082840237, + "grad_norm": 0.31744632203327944, + "kl": 0.0322265625, + "learning_rate": 7.100591715976331e-09, + "loss": 0.0013, + "reward": 0.1171875, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1678 + }, + { + "completion_length": 127.125, + "epoch": 0.993491124260355, + "grad_norm": 0.7658710762329471, + "kl": 0.03173828125, + "learning_rate": 6.50887573964497e-09, + "loss": 0.0013, + "reward": 0.1484375, + "reward_std": 0.09616719186306, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1484375, + "step": 1679 + }, + { + "completion_length": 127.140625, + "epoch": 0.9940828402366864, + "grad_norm": 0.31246663089019633, + "kl": 0.022216796875, + "learning_rate": 5.917159763313609e-09, + "loss": 0.0009, + "reward": 0.0234375, + "reward_std": 0.015625, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0234375, + "step": 1680 + }, + { + "completion_length": 117.453125, + "epoch": 0.9946745562130177, + "grad_norm": 0.6381105727226613, + "kl": 0.0277099609375, + "learning_rate": 5.325443786982248e-09, + "loss": 0.0011, + "reward": 0.0859375, + "reward_std": 0.078125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0859375, + "step": 1681 + }, + { + "completion_length": 126.5, + "epoch": 0.9952662721893492, + "grad_norm": 0.5154935070991707, + "kl": 0.031494140625, + "learning_rate": 4.733727810650887e-09, + "loss": 0.0013, + "reward": 0.171875, + "reward_std": 0.03125, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.171875, + "step": 1682 + }, + { + "completion_length": 125.3046875, + "epoch": 0.9958579881656805, + "grad_norm": 0.4533754572187416, + "kl": 0.028076171875, + "learning_rate": 4.142011834319526e-09, + "loss": 0.0011, + "reward": 0.1015625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1015625, + "step": 1683 + }, + { + "completion_length": 125.15625, + "epoch": 0.9964497041420118, + "grad_norm": 0.2569030935856801, + "kl": 0.0194091796875, + "learning_rate": 3.5502958579881654e-09, + "loss": 0.0008, + "reward": 0.046875, + "reward_std": 0.018042195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.046875, + "step": 1684 + }, + { + "completion_length": 125.9140625, + "epoch": 0.9970414201183432, + "grad_norm": 0.5811103968635605, + "kl": 0.029052734375, + "learning_rate": 2.9585798816568045e-09, + "loss": 0.0012, + "reward": 0.1171875, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1171875, + "step": 1685 + }, + { + "completion_length": 119.1328125, + "epoch": 0.9976331360946745, + "grad_norm": 0.5786173368302283, + "kl": 0.027099609375, + "learning_rate": 2.3668639053254436e-09, + "loss": 0.0011, + "reward": 0.1953125, + "reward_std": 0.046875, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.1953125, + "step": 1686 + }, + { + "completion_length": 125.03125, + "epoch": 0.9982248520710059, + "grad_norm": 0.6027847431992983, + "kl": 0.02294921875, + "learning_rate": 1.7751479289940827e-09, + "loss": 0.0009, + "reward": 0.0390625, + "reward_std": 0.033667195588350296, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0390625, + "step": 1687 + }, + { + "completion_length": 125.6484375, + "epoch": 0.9988165680473373, + "grad_norm": 0.05731473889859169, + "kl": 0.029296875, + "learning_rate": 1.1834319526627218e-09, + "loss": 0.0012, + "reward": 0.0625, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0625, + "step": 1688 + }, + { + "completion_length": 128.1796875, + "epoch": 0.9994082840236687, + "grad_norm": 0.4922073853722492, + "kl": 0.0283203125, + "learning_rate": 5.917159763313609e-10, + "loss": 0.0011, + "reward": 0.0703125, + "reward_std": 0.05170939117670059, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0703125, + "step": 1689 + }, + { + "completion_length": 82.0, + "epoch": 1.0, + "grad_norm": 0.6410215539358913, + "kl": 0.023681640625, + "learning_rate": 0.0, + "loss": 0.0011, + "reward": 0.0, + "reward_std": 0.0, + "rewards/format_reward": 0.0, + "rewards/message_judge_reward": 0.0, + "step": 1690 + } + ], + "logging_steps": 1.0, + "max_steps": 1690, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}