diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,8534 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.30012004801920766, + "eval_steps": 500, + "global_step": 250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17003.0, + "completions/max_terminated_length": 17003.0, + "completions/mean_length": 10512.40625, + "completions/mean_terminated_length": 10512.40625, + "completions/min_length": 6733.0, + "completions/min_terminated_length": 6733.0, + "entropy": 0.2287780987098813, + "epoch": 0.0012004801920768306, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3057364225387573, + "learning_rate": 1e-06, + "loss": 0.0187, + "num_tokens": 377149.0, + "reward": 2.172255277633667, + "reward_std": 2.8551955223083496, + "rewards/reward_func/mean": 2.1769778728485107, + "rewards/reward_func/std": 2.863737106323242, + "rewards/soft_overlong_punishment_reward/mean": -0.00472259521484375, + "rewards/soft_overlong_punishment_reward/std": 0.026715034618973732, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9920079708099365, + "sampling/importance_sampling_ratio/min": 0.06831115484237671, + "sampling/sampling_logp_difference/max": 2.6836822032928467, + "sampling/sampling_logp_difference/mean": 0.016660798341035843, + "step": 1, + "step_time": 260.7638783827424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13544.0, + "completions/max_terminated_length": 13544.0, + "completions/mean_length": 9758.78125, + "completions/mean_terminated_length": 9758.78125, + "completions/min_length": 7933.0, + "completions/min_terminated_length": 7933.0, + "entropy": 0.26108355820178986, + "epoch": 0.0024009603841536613, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45485132932662964, + "learning_rate": 9.959999999999999e-07, + "loss": -0.0342, + "num_tokens": 724854.0, + "reward": 2.3083786964416504, + "reward_std": 3.1393442153930664, + "rewards/reward_func/mean": 2.3083786964416504, + "rewards/reward_func/std": 3.1393442153930664, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9068528413772583, + "sampling/importance_sampling_ratio/mean": 0.9908677339553833, + "sampling/importance_sampling_ratio/min": 0.009255149401724339, + "sampling/sampling_logp_difference/max": 4.682575225830078, + "sampling/sampling_logp_difference/mean": 0.01871836744248867, + "step": 2, + "step_time": 251.4262609881116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14033.0, + "completions/max_terminated_length": 14033.0, + "completions/mean_length": 9934.3125, + "completions/mean_terminated_length": 9934.3125, + "completions/min_length": 4489.0, + "completions/min_terminated_length": 4489.0, + "entropy": 0.26445347908884287, + "epoch": 0.003601440576230492, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3962293267250061, + "learning_rate": 9.92e-07, + "loss": -0.0314, + "num_tokens": 1075280.0, + "reward": 2.6239943504333496, + "reward_std": 2.9723362922668457, + "rewards/reward_func/mean": 2.6239943504333496, + "rewards/reward_func/std": 2.972336769104004, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1135287284851074, + "sampling/importance_sampling_ratio/mean": 0.9906726479530334, + "sampling/importance_sampling_ratio/min": 0.20889143645763397, + "sampling/sampling_logp_difference/max": 1.5659406185150146, + "sampling/sampling_logp_difference/mean": 0.01895604282617569, + "step": 3, + "step_time": 210.0828171017347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15270.0, + "completions/max_terminated_length": 15270.0, + "completions/mean_length": 10381.21875, + "completions/mean_terminated_length": 10381.21875, + "completions/min_length": 5544.0, + "completions/min_terminated_length": 5544.0, + "entropy": 0.2782252887263894, + "epoch": 0.004801920768307323, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.33767879009246826, + "learning_rate": 9.88e-07, + "loss": -0.0169, + "num_tokens": 1430327.0, + "reward": 1775.4732666015625, + "reward_std": 4829.59228515625, + "rewards/reward_func/mean": 1775.4732666015625, + "rewards/reward_func/std": 4829.5927734375, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.093236207962036, + "sampling/importance_sampling_ratio/mean": 0.9902609586715698, + "sampling/importance_sampling_ratio/min": 0.18372339010238647, + "sampling/sampling_logp_difference/max": 1.694324016571045, + "sampling/sampling_logp_difference/mean": 0.019832005724310875, + "step": 4, + "step_time": 419.7083154738648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16885.0, + "completions/max_terminated_length": 16885.0, + "completions/mean_length": 12227.375, + "completions/mean_terminated_length": 12227.375, + "completions/min_length": 9054.0, + "completions/min_terminated_length": 9054.0, + "entropy": 0.2566723832860589, + "epoch": 0.006002400960384154, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3329413831233978, + "learning_rate": 9.84e-07, + "loss": 0.021, + "num_tokens": 1853891.0, + "reward": 1.8860834836959839, + "reward_std": 2.3241159915924072, + "rewards/reward_func/mean": 1.8933619260787964, + "rewards/reward_func/std": 2.3200485706329346, + "rewards/soft_overlong_punishment_reward/mean": -0.0072784423828125, + "rewards/soft_overlong_punishment_reward/std": 0.028263933956623077, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9909947514533997, + "sampling/importance_sampling_ratio/min": 0.17708024382591248, + "sampling/sampling_logp_difference/max": 1.7311522960662842, + "sampling/sampling_logp_difference/mean": 0.018662026152014732, + "step": 5, + "step_time": 244.05006601125933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16596.0, + "completions/max_terminated_length": 16596.0, + "completions/mean_length": 10645.0625, + "completions/mean_terminated_length": 10645.0625, + "completions/min_length": 7554.0, + "completions/min_terminated_length": 7554.0, + "entropy": 0.27365364972501993, + "epoch": 0.007202881152460984, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40234512090682983, + "learning_rate": 9.8e-07, + "loss": 0.0528, + "num_tokens": 2223725.0, + "reward": 1.9188847541809082, + "reward_std": 2.996480703353882, + "rewards/reward_func/mean": 1.9205021858215332, + "rewards/reward_func/std": 2.995396375656128, + "rewards/soft_overlong_punishment_reward/mean": -0.001617431640625, + "rewards/soft_overlong_punishment_reward/std": 0.009149574674665928, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9904695749282837, + "sampling/importance_sampling_ratio/min": 0.10882613807916641, + "sampling/sampling_logp_difference/max": 2.218003749847412, + "sampling/sampling_logp_difference/mean": 0.019550971686840057, + "step": 6, + "step_time": 222.7570838360116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16016.0, + "completions/max_terminated_length": 16016.0, + "completions/mean_length": 10611.75, + "completions/mean_terminated_length": 10611.75, + "completions/min_length": 3615.0, + "completions/min_terminated_length": 3615.0, + "entropy": 0.2584681212902069, + "epoch": 0.008403361344537815, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37323054671287537, + "learning_rate": 9.759999999999998e-07, + "loss": -0.034, + "num_tokens": 2600909.0, + "reward": 2.0009255409240723, + "reward_std": 2.1709108352661133, + "rewards/reward_func/mean": 2.0009255409240723, + "rewards/reward_func/std": 2.1709108352661133, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9910842776298523, + "sampling/importance_sampling_ratio/min": 0.12746387720108032, + "sampling/sampling_logp_difference/max": 2.059922218322754, + "sampling/sampling_logp_difference/mean": 0.018402449786663055, + "step": 7, + "step_time": 236.24530657555442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15769.0, + "completions/max_terminated_length": 15769.0, + "completions/mean_length": 11196.375, + "completions/mean_terminated_length": 11196.375, + "completions/min_length": 7706.0, + "completions/min_terminated_length": 7706.0, + "entropy": 0.2538974490016699, + "epoch": 0.009603841536614645, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.23562167584896088, + "learning_rate": 9.72e-07, + "loss": 0.0366, + "num_tokens": 2999913.0, + "reward": 0.09837960451841354, + "reward_std": 0.23358193039894104, + "rewards/reward_func/mean": 0.09837960451841354, + "rewards/reward_func/std": 0.23358193039894104, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9912012815475464, + "sampling/importance_sampling_ratio/min": 0.16663774847984314, + "sampling/sampling_logp_difference/max": 1.953153371810913, + "sampling/sampling_logp_difference/mean": 0.018111014738678932, + "step": 8, + "step_time": 234.31651349924505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15866.0, + "completions/max_terminated_length": 15866.0, + "completions/mean_length": 12010.46875, + "completions/mean_terminated_length": 12010.46875, + "completions/min_length": 8409.0, + "completions/min_terminated_length": 8409.0, + "entropy": 0.27975638769567013, + "epoch": 0.010804321728691477, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.376257061958313, + "learning_rate": 9.679999999999999e-07, + "loss": -0.0141, + "num_tokens": 3418312.0, + "reward": 3.2278504371643066, + "reward_std": 11.484737396240234, + "rewards/reward_func/mean": 3.2278504371643066, + "rewards/reward_func/std": 11.484737396240234, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.738037109375, + "sampling/importance_sampling_ratio/mean": 0.9900665879249573, + "sampling/importance_sampling_ratio/min": 0.0053693195804953575, + "sampling/sampling_logp_difference/max": 5.227054119110107, + "sampling/sampling_logp_difference/mean": 0.02012288197875023, + "step": 9, + "step_time": 239.36658373998944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17639.0, + "completions/max_terminated_length": 17639.0, + "completions/mean_length": 11850.59375, + "completions/mean_terminated_length": 11850.59375, + "completions/min_length": 8366.0, + "completions/min_terminated_length": 8366.0, + "entropy": 0.27660834789276123, + "epoch": 0.012004801920768308, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3692915141582489, + "learning_rate": 9.64e-07, + "loss": 0.0535, + "num_tokens": 3824779.0, + "reward": 0.4762687683105469, + "reward_std": 0.7091255784034729, + "rewards/reward_func/mean": 0.4858436584472656, + "rewards/reward_func/std": 0.7002295255661011, + "rewards/soft_overlong_punishment_reward/mean": -0.00957489013671875, + "rewards/soft_overlong_punishment_reward/std": 0.05416375771164894, + "sampling/importance_sampling_ratio/max": 2.2065083980560303, + "sampling/importance_sampling_ratio/mean": 0.9904176592826843, + "sampling/importance_sampling_ratio/min": 0.13561852276325226, + "sampling/sampling_logp_difference/max": 1.9979093074798584, + "sampling/sampling_logp_difference/mean": 0.019777128472924232, + "step": 10, + "step_time": 242.8805887659546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17650.0, + "completions/max_terminated_length": 17650.0, + "completions/mean_length": 11499.65625, + "completions/mean_terminated_length": 11499.65625, + "completions/min_length": 7873.0, + "completions/min_terminated_length": 7873.0, + "entropy": 0.24676325172185898, + "epoch": 0.013205282112845138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41307520866394043, + "learning_rate": 9.6e-07, + "loss": 0.0019, + "num_tokens": 4228592.0, + "reward": 1.2467007637023926, + "reward_std": 2.456725597381592, + "rewards/reward_func/mean": 1.256359577178955, + "rewards/reward_func/std": 2.451215982437134, + "rewards/soft_overlong_punishment_reward/mean": -0.0096588134765625, + "rewards/soft_overlong_punishment_reward/std": 0.05463850125670433, + "sampling/importance_sampling_ratio/max": 2.828012466430664, + "sampling/importance_sampling_ratio/mean": 0.9914248585700989, + "sampling/importance_sampling_ratio/min": 0.36373454332351685, + "sampling/sampling_logp_difference/max": 1.039574146270752, + "sampling/sampling_logp_difference/mean": 0.01774602197110653, + "step": 11, + "step_time": 265.6139151197858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14906.0, + "completions/max_terminated_length": 14906.0, + "completions/mean_length": 11677.15625, + "completions/mean_terminated_length": 11677.15625, + "completions/min_length": 7384.0, + "completions/min_terminated_length": 7384.0, + "entropy": 0.23969209752976894, + "epoch": 0.014405762304921969, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3561864495277405, + "learning_rate": 9.559999999999998e-07, + "loss": 0.0084, + "num_tokens": 4638621.0, + "reward": 0.6092424392700195, + "reward_std": 1.582262396812439, + "rewards/reward_func/mean": 0.6092424392700195, + "rewards/reward_func/std": 1.5822621583938599, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.7359044551849365, + "sampling/importance_sampling_ratio/mean": 0.9916994571685791, + "sampling/importance_sampling_ratio/min": 0.26353466510772705, + "sampling/sampling_logp_difference/max": 1.3335703611373901, + "sampling/sampling_logp_difference/mean": 0.017739253118634224, + "step": 12, + "step_time": 233.1278730876511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17077.0, + "completions/max_terminated_length": 17077.0, + "completions/mean_length": 10545.90625, + "completions/mean_terminated_length": 10545.90625, + "completions/min_length": 7467.0, + "completions/min_terminated_length": 7467.0, + "entropy": 0.2735821781679988, + "epoch": 0.015606242496998799, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3848613202571869, + "learning_rate": 9.52e-07, + "loss": 0.0584, + "num_tokens": 5005954.0, + "reward": 2.2471165657043457, + "reward_std": 3.020482301712036, + "rewards/reward_func/mean": 2.253136157989502, + "rewards/reward_func/std": 3.0157971382141113, + "rewards/soft_overlong_punishment_reward/mean": -0.00601959228515625, + "rewards/soft_overlong_punishment_reward/std": 0.030061684548854828, + "sampling/importance_sampling_ratio/max": 2.850409746170044, + "sampling/importance_sampling_ratio/mean": 0.9906243085861206, + "sampling/importance_sampling_ratio/min": 6.962251973163802e-06, + "sampling/sampling_logp_difference/max": 11.875007629394531, + "sampling/sampling_logp_difference/mean": 0.019261419773101807, + "step": 13, + "step_time": 243.16750305576716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14631.0, + "completions/max_terminated_length": 14631.0, + "completions/mean_length": 10139.75, + "completions/mean_terminated_length": 10139.75, + "completions/min_length": 5047.0, + "completions/min_terminated_length": 5047.0, + "entropy": 0.24060232006013393, + "epoch": 0.01680672268907563, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.383498877286911, + "learning_rate": 9.479999999999999e-07, + "loss": 0.0478, + "num_tokens": 5374362.0, + "reward": 0.869640588760376, + "reward_std": 1.4982999563217163, + "rewards/reward_func/mean": 0.869640588760376, + "rewards/reward_func/std": 1.4982999563217163, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.358623743057251, + "sampling/importance_sampling_ratio/mean": 0.9915570616722107, + "sampling/importance_sampling_ratio/min": 0.07955317199230194, + "sampling/sampling_logp_difference/max": 2.53132963180542, + "sampling/sampling_logp_difference/mean": 0.017563849687576294, + "step": 14, + "step_time": 217.46682468871586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17053.0, + "completions/max_terminated_length": 17053.0, + "completions/mean_length": 13305.40625, + "completions/mean_terminated_length": 13305.40625, + "completions/min_length": 10198.0, + "completions/min_terminated_length": 10198.0, + "entropy": 0.2278358107432723, + "epoch": 0.01800720288115246, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.29915085434913635, + "learning_rate": 9.439999999999999e-07, + "loss": 0.0318, + "num_tokens": 5840119.0, + "reward": 0.04119223356246948, + "reward_std": 0.2189895212650299, + "rewards/reward_func/mean": 0.04761618375778198, + "rewards/reward_func/std": 0.21552126109600067, + "rewards/soft_overlong_punishment_reward/mean": -0.0064239501953125, + "rewards/soft_overlong_punishment_reward/std": 0.029588615521788597, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9919904470443726, + "sampling/importance_sampling_ratio/min": 0.06755267828702927, + "sampling/sampling_logp_difference/max": 2.694847583770752, + "sampling/sampling_logp_difference/mean": 0.01704443246126175, + "step": 15, + "step_time": 473.42477186385076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16258.0, + "completions/max_terminated_length": 16258.0, + "completions/mean_length": 10728.21875, + "completions/mean_terminated_length": 10728.21875, + "completions/min_length": 6511.0, + "completions/min_terminated_length": 6511.0, + "entropy": 0.2541895778849721, + "epoch": 0.01920768307322929, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.357921302318573, + "learning_rate": 9.399999999999999e-07, + "loss": -0.0349, + "num_tokens": 6216630.0, + "reward": 3.3305792808532715, + "reward_std": 7.61651611328125, + "rewards/reward_func/mean": 3.3305792808532715, + "rewards/reward_func/std": 7.61651611328125, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.522019624710083, + "sampling/importance_sampling_ratio/mean": 0.9909427762031555, + "sampling/importance_sampling_ratio/min": 0.14655600488185883, + "sampling/sampling_logp_difference/max": 1.9203475713729858, + "sampling/sampling_logp_difference/mean": 0.018465207889676094, + "step": 16, + "step_time": 229.7186487779254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14954.0, + "completions/max_terminated_length": 14954.0, + "completions/mean_length": 10770.53125, + "completions/mean_terminated_length": 10770.53125, + "completions/min_length": 5937.0, + "completions/min_terminated_length": 5937.0, + "entropy": 0.2734638024121523, + "epoch": 0.02040816326530612, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.32195746898651123, + "learning_rate": 9.36e-07, + "loss": 0.0259, + "num_tokens": 6590607.0, + "reward": 0.6464040875434875, + "reward_std": 0.9011892676353455, + "rewards/reward_func/mean": 0.6464040875434875, + "rewards/reward_func/std": 0.9011892676353455, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1569271087646484, + "sampling/importance_sampling_ratio/mean": 0.9905099868774414, + "sampling/importance_sampling_ratio/min": 0.02047974243760109, + "sampling/sampling_logp_difference/max": 3.8883190155029297, + "sampling/sampling_logp_difference/mean": 0.019440319389104843, + "step": 17, + "step_time": 433.71242073585745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16352.0, + "completions/max_terminated_length": 16352.0, + "completions/mean_length": 9984.46875, + "completions/mean_terminated_length": 9984.46875, + "completions/min_length": 4147.0, + "completions/min_terminated_length": 4147.0, + "entropy": 0.2726652082055807, + "epoch": 0.021608643457382955, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3855082392692566, + "learning_rate": 9.32e-07, + "loss": 0.0234, + "num_tokens": 6944518.0, + "reward": 1.9245760440826416, + "reward_std": 2.4882936477661133, + "rewards/reward_func/mean": 1.9245760440826416, + "rewards/reward_func/std": 2.4882936477661133, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0238037109375, + "sampling/importance_sampling_ratio/mean": 0.9904158115386963, + "sampling/importance_sampling_ratio/min": 0.19708161056041718, + "sampling/sampling_logp_difference/max": 1.6241374015808105, + "sampling/sampling_logp_difference/mean": 0.01931804046034813, + "step": 18, + "step_time": 261.609183562221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19890.0, + "completions/max_terminated_length": 19890.0, + "completions/mean_length": 12736.40625, + "completions/mean_terminated_length": 12736.40625, + "completions/min_length": 8136.0, + "completions/min_terminated_length": 8136.0, + "entropy": 0.22564240265637636, + "epoch": 0.022809123649459785, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3456736207008362, + "learning_rate": 9.28e-07, + "loss": 0.0822, + "num_tokens": 7391411.0, + "reward": 55.75565719604492, + "reward_std": 127.06681823730469, + "rewards/reward_func/mean": 55.820682525634766, + "rewards/reward_func/std": 127.06536102294922, + "rewards/soft_overlong_punishment_reward/mean": -0.06502532958984375, + "rewards/soft_overlong_punishment_reward/std": 0.21553081274032593, + "sampling/importance_sampling_ratio/max": 2.4781954288482666, + "sampling/importance_sampling_ratio/mean": 0.9922029376029968, + "sampling/importance_sampling_ratio/min": 0.18941795825958252, + "sampling/sampling_logp_difference/max": 1.6637992858886719, + "sampling/sampling_logp_difference/mean": 0.016782237216830254, + "step": 19, + "step_time": 274.4253786377376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18974.0, + "completions/max_terminated_length": 18974.0, + "completions/mean_length": 11451.9375, + "completions/mean_terminated_length": 11451.9375, + "completions/min_length": 8223.0, + "completions/min_terminated_length": 8223.0, + "entropy": 0.24693564232438803, + "epoch": 0.024009603841536616, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37484318017959595, + "learning_rate": 9.24e-07, + "loss": 0.0537, + "num_tokens": 7801249.0, + "reward": 1.6996723413467407, + "reward_std": 2.6643435955047607, + "rewards/reward_func/mean": 1.7194324731826782, + "rewards/reward_func/std": 2.648789882659912, + "rewards/soft_overlong_punishment_reward/mean": -0.0197601318359375, + "rewards/soft_overlong_punishment_reward/std": 0.11178018897771835, + "sampling/importance_sampling_ratio/max": 2.0502030849456787, + "sampling/importance_sampling_ratio/mean": 0.991265594959259, + "sampling/importance_sampling_ratio/min": 0.03500676155090332, + "sampling/sampling_logp_difference/max": 3.3522140979766846, + "sampling/sampling_logp_difference/mean": 0.017859291285276413, + "step": 20, + "step_time": 254.13871470477898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14921.0, + "completions/max_terminated_length": 14921.0, + "completions/mean_length": 11261.625, + "completions/mean_terminated_length": 11261.625, + "completions/min_length": 8921.0, + "completions/min_terminated_length": 8921.0, + "entropy": 0.21324953343719244, + "epoch": 0.025210084033613446, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37738341093063354, + "learning_rate": 9.2e-07, + "loss": 0.0006, + "num_tokens": 8209717.0, + "reward": 5.487376689910889, + "reward_std": 5.091497421264648, + "rewards/reward_func/mean": 5.487376689910889, + "rewards/reward_func/std": 5.091497421264648, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.863360643386841, + "sampling/importance_sampling_ratio/mean": 0.9924338459968567, + "sampling/importance_sampling_ratio/min": 0.2230137288570404, + "sampling/sampling_logp_difference/max": 1.5005220174789429, + "sampling/sampling_logp_difference/mean": 0.01615520380437374, + "step": 21, + "step_time": 226.67027153016534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15897.0, + "completions/max_terminated_length": 15897.0, + "completions/mean_length": 9916.03125, + "completions/mean_terminated_length": 9916.03125, + "completions/min_length": 6559.0, + "completions/min_terminated_length": 6559.0, + "entropy": 0.2823543632403016, + "epoch": 0.026410564225690276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41728851199150085, + "learning_rate": 9.16e-07, + "loss": 0.0357, + "num_tokens": 8558022.0, + "reward": 1.0093857049942017, + "reward_std": 1.6929666996002197, + "rewards/reward_func/mean": 1.0093857049942017, + "rewards/reward_func/std": 1.6929665803909302, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9902706146240234, + "sampling/importance_sampling_ratio/min": 0.29672637581825256, + "sampling/sampling_logp_difference/max": 1.2743182182312012, + "sampling/sampling_logp_difference/mean": 0.020027777180075645, + "step": 22, + "step_time": 210.78112597286236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15085.0, + "completions/max_terminated_length": 15085.0, + "completions/mean_length": 9004.78125, + "completions/mean_terminated_length": 9004.78125, + "completions/min_length": 5019.0, + "completions/min_terminated_length": 5019.0, + "entropy": 0.26619420014321804, + "epoch": 0.027611044417767107, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4362182021141052, + "learning_rate": 9.12e-07, + "loss": -0.0028, + "num_tokens": 8867615.0, + "reward": 3.192936420440674, + "reward_std": 3.843594551086426, + "rewards/reward_func/mean": 3.192936420440674, + "rewards/reward_func/std": 3.8435938358306885, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.684079170227051, + "sampling/importance_sampling_ratio/mean": 0.9908168315887451, + "sampling/importance_sampling_ratio/min": 0.24921053647994995, + "sampling/sampling_logp_difference/max": 1.3894572257995605, + "sampling/sampling_logp_difference/mean": 0.019214916974306107, + "step": 23, + "step_time": 200.50691007694695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15050.0, + "completions/max_terminated_length": 15050.0, + "completions/mean_length": 10668.5625, + "completions/mean_terminated_length": 10668.5625, + "completions/min_length": 6005.0, + "completions/min_terminated_length": 6005.0, + "entropy": 0.2756733503192663, + "epoch": 0.028811524609843937, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37513336539268494, + "learning_rate": 9.08e-07, + "loss": 0.0388, + "num_tokens": 9237721.0, + "reward": 0.4544358551502228, + "reward_std": 0.5964015126228333, + "rewards/reward_func/mean": 0.4544358551502228, + "rewards/reward_func/std": 0.5964015126228333, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.185565233230591, + "sampling/importance_sampling_ratio/mean": 0.9904376268386841, + "sampling/importance_sampling_ratio/min": 0.3395426869392395, + "sampling/sampling_logp_difference/max": 1.080155611038208, + "sampling/sampling_logp_difference/mean": 0.019677145406603813, + "step": 24, + "step_time": 225.78321035497356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14237.0, + "completions/max_terminated_length": 14237.0, + "completions/mean_length": 10287.375, + "completions/mean_terminated_length": 10287.375, + "completions/min_length": 6909.0, + "completions/min_terminated_length": 6909.0, + "entropy": 0.2614977052435279, + "epoch": 0.030012004801920768, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3477057218551636, + "learning_rate": 9.039999999999999e-07, + "loss": 0.0355, + "num_tokens": 9600461.0, + "reward": 1.3245511054992676, + "reward_std": 2.4857571125030518, + "rewards/reward_func/mean": 1.3245511054992676, + "rewards/reward_func/std": 2.4857571125030518, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.117020606994629, + "sampling/importance_sampling_ratio/mean": 0.9908631443977356, + "sampling/importance_sampling_ratio/min": 0.12998507916927338, + "sampling/sampling_logp_difference/max": 2.0403356552124023, + "sampling/sampling_logp_difference/mean": 0.018762774765491486, + "step": 25, + "step_time": 207.05053522088565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14980.0, + "completions/max_terminated_length": 14980.0, + "completions/mean_length": 9842.53125, + "completions/mean_terminated_length": 9842.53125, + "completions/min_length": 6382.0, + "completions/min_terminated_length": 6382.0, + "entropy": 0.2648027054965496, + "epoch": 0.031212484993997598, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3818010091781616, + "learning_rate": 9e-07, + "loss": 0.0235, + "num_tokens": 9945774.0, + "reward": 1.9594688415527344, + "reward_std": 1.9573324918746948, + "rewards/reward_func/mean": 1.9594688415527344, + "rewards/reward_func/std": 1.9573323726654053, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.115938425064087, + "sampling/importance_sampling_ratio/mean": 0.9907746315002441, + "sampling/importance_sampling_ratio/min": 0.16936549544334412, + "sampling/sampling_logp_difference/max": 1.7756961584091187, + "sampling/sampling_logp_difference/mean": 0.018964044749736786, + "step": 26, + "step_time": 210.5417184109101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14752.0, + "completions/max_terminated_length": 14752.0, + "completions/mean_length": 10530.3125, + "completions/mean_terminated_length": 10530.3125, + "completions/min_length": 6272.0, + "completions/min_terminated_length": 6272.0, + "entropy": 0.24934661202132702, + "epoch": 0.03241296518607443, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40029430389404297, + "learning_rate": 8.96e-07, + "loss": -0.0036, + "num_tokens": 10312320.0, + "reward": 3.0405454635620117, + "reward_std": 3.8312883377075195, + "rewards/reward_func/mean": 3.0405454635620117, + "rewards/reward_func/std": 3.8312880992889404, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9914329648017883, + "sampling/importance_sampling_ratio/min": 0.09976457804441452, + "sampling/sampling_logp_difference/max": 2.3049421310424805, + "sampling/sampling_logp_difference/mean": 0.017998939380049706, + "step": 27, + "step_time": 431.60967801115476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17459.0, + "completions/max_terminated_length": 17459.0, + "completions/mean_length": 11231.8125, + "completions/mean_terminated_length": 11231.8125, + "completions/min_length": 5633.0, + "completions/min_terminated_length": 5633.0, + "entropy": 0.25841675139963627, + "epoch": 0.03361344537815126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3741910755634308, + "learning_rate": 8.92e-07, + "loss": 0.0071, + "num_tokens": 10709450.0, + "reward": 1.4909887313842773, + "reward_std": 2.263838291168213, + "rewards/reward_func/mean": 1.499190330505371, + "rewards/reward_func/std": 2.2577483654022217, + "rewards/soft_overlong_punishment_reward/mean": -0.00820159912109375, + "rewards/soft_overlong_punishment_reward/std": 0.04639524966478348, + "sampling/importance_sampling_ratio/max": 2.4496078491210938, + "sampling/importance_sampling_ratio/mean": 0.99090176820755, + "sampling/importance_sampling_ratio/min": 0.13220876455307007, + "sampling/sampling_logp_difference/max": 2.0233731269836426, + "sampling/sampling_logp_difference/mean": 0.01868780329823494, + "step": 28, + "step_time": 426.57934639882296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16356.0, + "completions/max_terminated_length": 16356.0, + "completions/mean_length": 9573.6875, + "completions/mean_terminated_length": 9573.6875, + "completions/min_length": 2818.0, + "completions/min_terminated_length": 2818.0, + "entropy": 0.2775294054299593, + "epoch": 0.03481392557022809, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3293408453464508, + "learning_rate": 8.88e-07, + "loss": 0.0192, + "num_tokens": 11044352.0, + "reward": 1.3649942874908447, + "reward_std": 1.9040188789367676, + "rewards/reward_func/mean": 1.3649942874908447, + "rewards/reward_func/std": 1.9040188789367676, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.990324854850769, + "sampling/importance_sampling_ratio/min": 0.37744393944740295, + "sampling/sampling_logp_difference/max": 1.3116214275360107, + "sampling/sampling_logp_difference/mean": 0.019651668146252632, + "step": 29, + "step_time": 238.1490348140942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14524.0, + "completions/max_terminated_length": 14524.0, + "completions/mean_length": 11151.8125, + "completions/mean_terminated_length": 11151.8125, + "completions/min_length": 7531.0, + "completions/min_terminated_length": 7531.0, + "entropy": 0.24554751068353653, + "epoch": 0.03601440576230492, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36097458004951477, + "learning_rate": 8.839999999999999e-07, + "loss": -0.0026, + "num_tokens": 11442354.0, + "reward": 2.1457722187042236, + "reward_std": 2.174523115158081, + "rewards/reward_func/mean": 2.1457722187042236, + "rewards/reward_func/std": 2.17452335357666, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9914038777351379, + "sampling/importance_sampling_ratio/min": 0.1909109503030777, + "sampling/sampling_logp_difference/max": 1.6559481620788574, + "sampling/sampling_logp_difference/mean": 0.0179743692278862, + "step": 30, + "step_time": 251.35512228007428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13663.0, + "completions/max_terminated_length": 13663.0, + "completions/mean_length": 8797.78125, + "completions/mean_terminated_length": 8797.78125, + "completions/min_length": 3457.0, + "completions/min_terminated_length": 3457.0, + "entropy": 0.26490489952266216, + "epoch": 0.03721488595438175, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45504486560821533, + "learning_rate": 8.799999999999999e-07, + "loss": 0.0903, + "num_tokens": 11752779.0, + "reward": 2.032346725463867, + "reward_std": 2.6423678398132324, + "rewards/reward_func/mean": 2.032346725463867, + "rewards/reward_func/std": 2.6423676013946533, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9907644987106323, + "sampling/importance_sampling_ratio/min": 0.12091671675443649, + "sampling/sampling_logp_difference/max": 2.1126532554626465, + "sampling/sampling_logp_difference/mean": 0.01906166598200798, + "step": 31, + "step_time": 210.08964343706612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 17167.0, + "completions/mean_length": 10623.3125, + "completions/mean_terminated_length": 10305.3544921875, + "completions/min_length": 4879.0, + "completions/min_terminated_length": 4879.0, + "entropy": 0.2628009468317032, + "epoch": 0.03841536614645858, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.2900553047657013, + "learning_rate": 8.76e-07, + "loss": -0.0423, + "num_tokens": 12126181.0, + "reward": 1.0021374225616455, + "reward_std": 1.9135971069335938, + "rewards/reward_func/mean": 1.0393612384796143, + "rewards/reward_func/std": 1.8841371536254883, + "rewards/soft_overlong_punishment_reward/mean": -0.03722381591796875, + "rewards/soft_overlong_punishment_reward/std": 0.17890377342700958, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9907552599906921, + "sampling/importance_sampling_ratio/min": 0.2805570363998413, + "sampling/sampling_logp_difference/max": 1.2783308029174805, + "sampling/sampling_logp_difference/mean": 0.018829116597771645, + "step": 32, + "step_time": 280.71025710599497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14347.0, + "completions/max_terminated_length": 14347.0, + "completions/mean_length": 10457.4375, + "completions/mean_terminated_length": 10457.4375, + "completions/min_length": 7484.0, + "completions/min_terminated_length": 7484.0, + "entropy": 0.28439216688275337, + "epoch": 0.03961584633853541, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3339356482028961, + "learning_rate": 8.72e-07, + "loss": 0.031, + "num_tokens": 12492219.0, + "reward": 0.7305504083633423, + "reward_std": 0.7552641034126282, + "rewards/reward_func/mean": 0.7305504083633423, + "rewards/reward_func/std": 0.7552641034126282, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9899792671203613, + "sampling/importance_sampling_ratio/min": 0.15883956849575043, + "sampling/sampling_logp_difference/max": 1.8398605585098267, + "sampling/sampling_logp_difference/mean": 0.020156892016530037, + "step": 33, + "step_time": 222.89753303222824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14362.0, + "completions/max_terminated_length": 14362.0, + "completions/mean_length": 10071.5, + "completions/mean_terminated_length": 10071.5, + "completions/min_length": 6369.0, + "completions/min_terminated_length": 6369.0, + "entropy": 0.24872859101742506, + "epoch": 0.04081632653061224, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4110364019870758, + "learning_rate": 8.68e-07, + "loss": -0.0135, + "num_tokens": 12855851.0, + "reward": 2.7421875, + "reward_std": 3.5190269947052, + "rewards/reward_func/mean": 2.7421875, + "rewards/reward_func/std": 3.5190269947052, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9915586113929749, + "sampling/importance_sampling_ratio/min": 0.27868548035621643, + "sampling/sampling_logp_difference/max": 1.2776713371276855, + "sampling/sampling_logp_difference/mean": 0.01771131530404091, + "step": 34, + "step_time": 224.99640086409636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13259.0, + "completions/max_terminated_length": 13259.0, + "completions/mean_length": 8307.40625, + "completions/mean_terminated_length": 8307.40625, + "completions/min_length": 3245.0, + "completions/min_terminated_length": 3245.0, + "entropy": 0.2815095093101263, + "epoch": 0.04201680672268908, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4219931364059448, + "learning_rate": 8.639999999999999e-07, + "loss": 0.02, + "num_tokens": 13148672.0, + "reward": 1.6012442111968994, + "reward_std": 2.0349299907684326, + "rewards/reward_func/mean": 1.6012442111968994, + "rewards/reward_func/std": 2.0349299907684326, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3349249362945557, + "sampling/importance_sampling_ratio/mean": 0.990214467048645, + "sampling/importance_sampling_ratio/min": 0.10321995615959167, + "sampling/sampling_logp_difference/max": 2.270893096923828, + "sampling/sampling_logp_difference/mean": 0.019673598930239677, + "step": 35, + "step_time": 197.53019676276017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13594.0, + "completions/max_terminated_length": 13594.0, + "completions/mean_length": 10127.0625, + "completions/mean_terminated_length": 10127.0625, + "completions/min_length": 6623.0, + "completions/min_terminated_length": 6623.0, + "entropy": 0.2668821206316352, + "epoch": 0.04321728691476591, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37350013852119446, + "learning_rate": 8.599999999999999e-07, + "loss": 0.0004, + "num_tokens": 13502514.0, + "reward": 2.3392128944396973, + "reward_std": 3.019270896911621, + "rewards/reward_func/mean": 2.3392128944396973, + "rewards/reward_func/std": 3.019270896911621, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.516148328781128, + "sampling/importance_sampling_ratio/mean": 0.9906859397888184, + "sampling/importance_sampling_ratio/min": 0.2636016309261322, + "sampling/sampling_logp_difference/max": 1.3333163261413574, + "sampling/sampling_logp_difference/mean": 0.01926989108324051, + "step": 36, + "step_time": 219.1794932379853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15015.0, + "completions/max_terminated_length": 15015.0, + "completions/mean_length": 10019.1875, + "completions/mean_terminated_length": 10019.1875, + "completions/min_length": 6810.0, + "completions/min_terminated_length": 6810.0, + "entropy": 0.24516229517757893, + "epoch": 0.04441776710684274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37556055188179016, + "learning_rate": 8.559999999999999e-07, + "loss": -0.0232, + "num_tokens": 13860184.0, + "reward": 5.546761512756348, + "reward_std": 4.9547014236450195, + "rewards/reward_func/mean": 5.546761512756348, + "rewards/reward_func/std": 4.954700946807861, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9915067553520203, + "sampling/importance_sampling_ratio/min": 0.3167104423046112, + "sampling/sampling_logp_difference/max": 1.2001211643218994, + "sampling/sampling_logp_difference/mean": 0.017740583047270775, + "step": 37, + "step_time": 429.79983439377975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15132.0, + "completions/max_terminated_length": 15132.0, + "completions/mean_length": 10126.53125, + "completions/mean_terminated_length": 10126.53125, + "completions/min_length": 3623.0, + "completions/min_terminated_length": 3623.0, + "entropy": 0.2536043291911483, + "epoch": 0.04561824729891957, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4053056538105011, + "learning_rate": 8.52e-07, + "loss": 0.0016, + "num_tokens": 14219801.0, + "reward": 1.1494144201278687, + "reward_std": 1.515267252922058, + "rewards/reward_func/mean": 1.1494144201278687, + "rewards/reward_func/std": 1.515267252922058, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.504192590713501, + "sampling/importance_sampling_ratio/mean": 0.9913269281387329, + "sampling/importance_sampling_ratio/min": 0.11853170394897461, + "sampling/sampling_logp_difference/max": 2.1325747966766357, + "sampling/sampling_logp_difference/mean": 0.018079929053783417, + "step": 38, + "step_time": 231.09112938609906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13953.0, + "completions/max_terminated_length": 13953.0, + "completions/mean_length": 9400.75, + "completions/mean_terminated_length": 9400.75, + "completions/min_length": 3400.0, + "completions/min_terminated_length": 3400.0, + "entropy": 0.2656275276094675, + "epoch": 0.0468187274909964, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.35668453574180603, + "learning_rate": 8.48e-07, + "loss": 0.0444, + "num_tokens": 14552409.0, + "reward": 1.375227689743042, + "reward_std": 2.0771355628967285, + "rewards/reward_func/mean": 1.375227689743042, + "rewards/reward_func/std": 2.0771355628967285, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.8366079330444336, + "sampling/importance_sampling_ratio/mean": 0.9909170269966125, + "sampling/importance_sampling_ratio/min": 0.021294279024004936, + "sampling/sampling_logp_difference/max": 3.8493168354034424, + "sampling/sampling_logp_difference/mean": 0.018923122435808182, + "step": 39, + "step_time": 205.99593690189067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14127.0, + "completions/max_terminated_length": 14127.0, + "completions/mean_length": 9623.75, + "completions/mean_terminated_length": 9623.75, + "completions/min_length": 5045.0, + "completions/min_terminated_length": 5045.0, + "entropy": 0.22837793547660112, + "epoch": 0.04801920768307323, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.344384104013443, + "learning_rate": 8.439999999999999e-07, + "loss": 0.0052, + "num_tokens": 14898929.0, + "reward": 0.5646069049835205, + "reward_std": 1.1903877258300781, + "rewards/reward_func/mean": 0.5646069049835205, + "rewards/reward_func/std": 1.1903877258300781, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.5692195892333984, + "sampling/importance_sampling_ratio/mean": 0.9922388195991516, + "sampling/importance_sampling_ratio/min": 0.14704354107379913, + "sampling/sampling_logp_difference/max": 1.9170265197753906, + "sampling/sampling_logp_difference/mean": 0.01625021919608116, + "step": 40, + "step_time": 228.8796401922591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16700.0, + "completions/max_terminated_length": 16700.0, + "completions/mean_length": 11196.0, + "completions/mean_terminated_length": 11196.0, + "completions/min_length": 7400.0, + "completions/min_terminated_length": 7400.0, + "entropy": 0.23109178990125656, + "epoch": 0.04921968787515006, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35686036944389343, + "learning_rate": 8.399999999999999e-07, + "loss": -0.025, + "num_tokens": 15295537.0, + "reward": 1.7151178121566772, + "reward_std": 2.868417501449585, + "rewards/reward_func/mean": 1.7175287008285522, + "rewards/reward_func/std": 2.868037700653076, + "rewards/soft_overlong_punishment_reward/mean": -0.002410888671875, + "rewards/soft_overlong_punishment_reward/std": 0.013638045638799667, + "sampling/importance_sampling_ratio/max": 2.9357402324676514, + "sampling/importance_sampling_ratio/mean": 0.9918723106384277, + "sampling/importance_sampling_ratio/min": 0.3307071328163147, + "sampling/sampling_logp_difference/max": 1.1065220832824707, + "sampling/sampling_logp_difference/mean": 0.016960658133029938, + "step": 41, + "step_time": 247.38727062719408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11833.0, + "completions/max_terminated_length": 11833.0, + "completions/mean_length": 9130.28125, + "completions/mean_terminated_length": 9130.28125, + "completions/min_length": 6003.0, + "completions/min_terminated_length": 6003.0, + "entropy": 0.27010741643607616, + "epoch": 0.05042016806722689, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.36788010597229004, + "learning_rate": 8.359999999999999e-07, + "loss": 0.0177, + "num_tokens": 15624346.0, + "reward": 0.4658914804458618, + "reward_std": 0.9191152453422546, + "rewards/reward_func/mean": 0.4658914804458618, + "rewards/reward_func/std": 0.9191153049468994, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9905910491943359, + "sampling/importance_sampling_ratio/min": 0.02883448265492916, + "sampling/sampling_logp_difference/max": 3.5461833477020264, + "sampling/sampling_logp_difference/mean": 0.01935204118490219, + "step": 42, + "step_time": 186.53937864280306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15715.0, + "completions/max_terminated_length": 15715.0, + "completions/mean_length": 11385.625, + "completions/mean_terminated_length": 11385.625, + "completions/min_length": 5736.0, + "completions/min_terminated_length": 5736.0, + "entropy": 0.2562467288225889, + "epoch": 0.05162064825930372, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.31052911281585693, + "learning_rate": 8.319999999999999e-07, + "loss": -0.0051, + "num_tokens": 16022038.0, + "reward": 1.6798204183578491, + "reward_std": 2.83245587348938, + "rewards/reward_func/mean": 1.6798204183578491, + "rewards/reward_func/std": 2.83245587348938, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.5458321571350098, + "sampling/importance_sampling_ratio/mean": 0.9909483790397644, + "sampling/importance_sampling_ratio/min": 0.2335936576128006, + "sampling/sampling_logp_difference/max": 1.454172134399414, + "sampling/sampling_logp_difference/mean": 0.01847730576992035, + "step": 43, + "step_time": 446.1300635159714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17778.0, + "completions/max_terminated_length": 17778.0, + "completions/mean_length": 9443.03125, + "completions/mean_terminated_length": 9443.03125, + "completions/min_length": 4794.0, + "completions/min_terminated_length": 4794.0, + "entropy": 0.25649653095752, + "epoch": 0.05282112845138055, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4464053809642792, + "learning_rate": 8.28e-07, + "loss": 0.0453, + "num_tokens": 16358935.0, + "reward": 1.2445961236953735, + "reward_std": 2.735797643661499, + "rewards/reward_func/mean": 1.255231499671936, + "rewards/reward_func/std": 2.730092763900757, + "rewards/soft_overlong_punishment_reward/mean": -0.0106353759765625, + "rewards/soft_overlong_punishment_reward/std": 0.060162775218486786, + "sampling/importance_sampling_ratio/max": 2.382478713989258, + "sampling/importance_sampling_ratio/mean": 0.9911012649536133, + "sampling/importance_sampling_ratio/min": 0.2831081748008728, + "sampling/sampling_logp_difference/max": 1.2619261741638184, + "sampling/sampling_logp_difference/mean": 0.018203921616077423, + "step": 44, + "step_time": 238.45520115306135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 18164.0, + "completions/max_terminated_length": 18164.0, + "completions/mean_length": 11039.0625, + "completions/mean_terminated_length": 11039.0625, + "completions/min_length": 6961.0, + "completions/min_terminated_length": 6961.0, + "entropy": 0.26013822108507156, + "epoch": 0.05402160864345738, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.7753751873970032, + "learning_rate": 8.24e-07, + "loss": 0.0627, + "num_tokens": 16746577.0, + "reward": 0.7348491549491882, + "reward_std": 1.1186299324035645, + "rewards/reward_func/mean": 0.756905734539032, + "rewards/reward_func/std": 1.0995012521743774, + "rewards/soft_overlong_punishment_reward/mean": -0.02205657958984375, + "rewards/soft_overlong_punishment_reward/std": 0.08923598378896713, + "sampling/importance_sampling_ratio/max": 2.8484528064727783, + "sampling/importance_sampling_ratio/mean": 0.9910010695457458, + "sampling/importance_sampling_ratio/min": 0.23139524459838867, + "sampling/sampling_logp_difference/max": 1.4636280536651611, + "sampling/sampling_logp_difference/mean": 0.01876596175134182, + "step": 45, + "step_time": 266.24963577114977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16171.0, + "completions/max_terminated_length": 16171.0, + "completions/mean_length": 10262.96875, + "completions/mean_terminated_length": 10262.96875, + "completions/min_length": 5913.0, + "completions/min_terminated_length": 5913.0, + "entropy": 0.25811031460762024, + "epoch": 0.055222088835534214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.378570556640625, + "learning_rate": 8.199999999999999e-07, + "loss": 0.0386, + "num_tokens": 17105832.0, + "reward": 2.625810146331787, + "reward_std": 3.4443087577819824, + "rewards/reward_func/mean": 2.625810146331787, + "rewards/reward_func/std": 3.4443087577819824, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0112507343292236, + "sampling/importance_sampling_ratio/mean": 0.9910373091697693, + "sampling/importance_sampling_ratio/min": 0.15773901343345642, + "sampling/sampling_logp_difference/max": 1.846813440322876, + "sampling/sampling_logp_difference/mean": 0.01831601746380329, + "step": 46, + "step_time": 237.72815251280554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15380.0, + "completions/max_terminated_length": 15380.0, + "completions/mean_length": 8343.25, + "completions/mean_terminated_length": 8343.25, + "completions/min_length": 4085.0, + "completions/min_terminated_length": 4085.0, + "entropy": 0.264944227412343, + "epoch": 0.056422569027611044, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4716252088546753, + "learning_rate": 8.159999999999999e-07, + "loss": 0.0628, + "num_tokens": 17399992.0, + "reward": 2.4003467559814453, + "reward_std": 3.171684741973877, + "rewards/reward_func/mean": 2.4003467559814453, + "rewards/reward_func/std": 3.171684503555298, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.402794599533081, + "sampling/importance_sampling_ratio/mean": 0.9908208250999451, + "sampling/importance_sampling_ratio/min": 0.3649902939796448, + "sampling/sampling_logp_difference/max": 1.0078845024108887, + "sampling/sampling_logp_difference/mean": 0.018845025449991226, + "step": 47, + "step_time": 209.52517059491947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14507.0, + "completions/max_terminated_length": 14507.0, + "completions/mean_length": 10999.65625, + "completions/mean_terminated_length": 10999.65625, + "completions/min_length": 7392.0, + "completions/min_terminated_length": 7392.0, + "entropy": 0.2484130971133709, + "epoch": 0.057623049219687875, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3668311834335327, + "learning_rate": 8.12e-07, + "loss": -0.0103, + "num_tokens": 17789653.0, + "reward": 0.8897507786750793, + "reward_std": 0.6408941149711609, + "rewards/reward_func/mean": 0.8897507786750793, + "rewards/reward_func/std": 0.6408940553665161, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.341449022293091, + "sampling/importance_sampling_ratio/mean": 0.9913567304611206, + "sampling/importance_sampling_ratio/min": 0.022933995351195335, + "sampling/sampling_logp_difference/max": 3.775135040283203, + "sampling/sampling_logp_difference/mean": 0.017700741067528725, + "step": 48, + "step_time": 227.09850537101738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16482.0, + "completions/max_terminated_length": 16482.0, + "completions/mean_length": 11000.375, + "completions/mean_terminated_length": 11000.375, + "completions/min_length": 6194.0, + "completions/min_terminated_length": 6194.0, + "entropy": 0.2574823070317507, + "epoch": 0.058823529411764705, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37714579701423645, + "learning_rate": 8.08e-07, + "loss": 0.039, + "num_tokens": 18186617.0, + "reward": 1.1769545078277588, + "reward_std": 0.6496437788009644, + "rewards/reward_func/mean": 1.1777021884918213, + "rewards/reward_func/std": 0.6509500741958618, + "rewards/soft_overlong_punishment_reward/mean": -0.0007476806640625, + "rewards/soft_overlong_punishment_reward/std": 0.004229520447552204, + "sampling/importance_sampling_ratio/max": 2.0076770782470703, + "sampling/importance_sampling_ratio/mean": 0.9911171197891235, + "sampling/importance_sampling_ratio/min": 0.015267138369381428, + "sampling/sampling_logp_difference/max": 4.1820526123046875, + "sampling/sampling_logp_difference/mean": 0.017983591184020042, + "step": 49, + "step_time": 450.96922948525753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12987.0, + "completions/max_terminated_length": 12987.0, + "completions/mean_length": 9108.96875, + "completions/mean_terminated_length": 9108.96875, + "completions/min_length": 5579.0, + "completions/min_terminated_length": 5579.0, + "entropy": 0.29258614405989647, + "epoch": 0.060024009603841535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4107678234577179, + "learning_rate": 8.04e-07, + "loss": -0.0154, + "num_tokens": 18501752.0, + "reward": 1.3277008533477783, + "reward_std": 2.548473596572876, + "rewards/reward_func/mean": 1.3277008533477783, + "rewards/reward_func/std": 2.548473358154297, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.7192769050598145, + "sampling/importance_sampling_ratio/mean": 0.9897809624671936, + "sampling/importance_sampling_ratio/min": 0.033950600773096085, + "sampling/sampling_logp_difference/max": 3.3828487396240234, + "sampling/sampling_logp_difference/mean": 0.020536189898848534, + "step": 50, + "step_time": 202.79201423272025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15226.0, + "completions/max_terminated_length": 15226.0, + "completions/mean_length": 8895.78125, + "completions/mean_terminated_length": 8895.78125, + "completions/min_length": 3790.0, + "completions/min_terminated_length": 3790.0, + "entropy": 0.24668313097208738, + "epoch": 0.061224489795918366, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4362691640853882, + "learning_rate": 8e-07, + "loss": -0.015, + "num_tokens": 18819793.0, + "reward": 4.881908893585205, + "reward_std": 4.377172946929932, + "rewards/reward_func/mean": 4.881908893585205, + "rewards/reward_func/std": 4.377172470092773, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1437125205993652, + "sampling/importance_sampling_ratio/mean": 0.991713285446167, + "sampling/importance_sampling_ratio/min": 0.0913717970252037, + "sampling/sampling_logp_difference/max": 2.3928184509277344, + "sampling/sampling_logp_difference/mean": 0.0172872394323349, + "step": 51, + "step_time": 230.47769821400288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10907.0, + "completions/max_terminated_length": 10907.0, + "completions/mean_length": 8155.625, + "completions/mean_terminated_length": 8155.625, + "completions/min_length": 4708.0, + "completions/min_terminated_length": 4708.0, + "entropy": 0.2947898115962744, + "epoch": 0.062424969987995196, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3665950894355774, + "learning_rate": 7.96e-07, + "loss": 0.0084, + "num_tokens": 19104357.0, + "reward": 2.1094980239868164, + "reward_std": 2.361452102661133, + "rewards/reward_func/mean": 2.1094980239868164, + "rewards/reward_func/std": 2.361452102661133, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.565842866897583, + "sampling/importance_sampling_ratio/mean": 0.9897834062576294, + "sampling/importance_sampling_ratio/min": 0.222982257604599, + "sampling/sampling_logp_difference/max": 1.500663161277771, + "sampling/sampling_logp_difference/mean": 0.02075599506497383, + "step": 52, + "step_time": 177.9754405770218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19155.0, + "completions/max_terminated_length": 19155.0, + "completions/mean_length": 10726.125, + "completions/mean_terminated_length": 10726.125, + "completions/min_length": 5744.0, + "completions/min_terminated_length": 5744.0, + "entropy": 0.2794426204636693, + "epoch": 0.06362545018007203, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.34014397859573364, + "learning_rate": 7.92e-07, + "loss": 0.051, + "num_tokens": 19475873.0, + "reward": 0.08617197722196579, + "reward_std": 0.35582923889160156, + "rewards/reward_func/mean": 0.10731302201747894, + "rewards/reward_func/std": 0.3313770294189453, + "rewards/soft_overlong_punishment_reward/mean": -0.02114105224609375, + "rewards/soft_overlong_punishment_reward/std": 0.1195918545126915, + "sampling/importance_sampling_ratio/max": 2.9801175594329834, + "sampling/importance_sampling_ratio/mean": 0.990292489528656, + "sampling/importance_sampling_ratio/min": 0.1588689088821411, + "sampling/sampling_logp_difference/max": 1.8396759033203125, + "sampling/sampling_logp_difference/mean": 0.019603673368692398, + "step": 53, + "step_time": 264.9613647001097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14962.0, + "completions/max_terminated_length": 14962.0, + "completions/mean_length": 10665.8125, + "completions/mean_terminated_length": 10665.8125, + "completions/min_length": 5778.0, + "completions/min_terminated_length": 5778.0, + "entropy": 0.2630910240113735, + "epoch": 0.06482593037214886, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36794134974479675, + "learning_rate": 7.88e-07, + "loss": 0.0013, + "num_tokens": 19851875.0, + "reward": 1.1953718662261963, + "reward_std": 2.0580642223358154, + "rewards/reward_func/mean": 1.1953718662261963, + "rewards/reward_func/std": 2.0580642223358154, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.635934829711914, + "sampling/importance_sampling_ratio/mean": 0.9908843040466309, + "sampling/importance_sampling_ratio/min": 0.25040531158447266, + "sampling/sampling_logp_difference/max": 1.3846744298934937, + "sampling/sampling_logp_difference/mean": 0.01858539693057537, + "step": 54, + "step_time": 230.69495908927638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13381.0, + "completions/max_terminated_length": 13381.0, + "completions/mean_length": 8781.25, + "completions/mean_terminated_length": 8781.25, + "completions/min_length": 3175.0, + "completions/min_terminated_length": 3175.0, + "entropy": 0.2748223738744855, + "epoch": 0.06602641056422569, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42096376419067383, + "learning_rate": 7.84e-07, + "loss": -0.0035, + "num_tokens": 20161883.0, + "reward": 0.9055551290512085, + "reward_std": 0.9041584134101868, + "rewards/reward_func/mean": 0.9055551290512085, + "rewards/reward_func/std": 0.9041584730148315, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9826617240905762, + "sampling/importance_sampling_ratio/mean": 0.9905050992965698, + "sampling/importance_sampling_ratio/min": 0.18111246824264526, + "sampling/sampling_logp_difference/max": 1.708636999130249, + "sampling/sampling_logp_difference/mean": 0.01938163861632347, + "step": 55, + "step_time": 206.5735895021353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12658.0, + "completions/max_terminated_length": 12658.0, + "completions/mean_length": 8712.09375, + "completions/mean_terminated_length": 8712.09375, + "completions/min_length": 5680.0, + "completions/min_terminated_length": 5680.0, + "entropy": 0.2725402005016804, + "epoch": 0.06722689075630252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4232986569404602, + "learning_rate": 7.799999999999999e-07, + "loss": 0.0346, + "num_tokens": 20467166.0, + "reward": 1.345833659172058, + "reward_std": 2.5527396202087402, + "rewards/reward_func/mean": 1.345833659172058, + "rewards/reward_func/std": 2.552739381790161, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3792266845703125, + "sampling/importance_sampling_ratio/mean": 0.9905150532722473, + "sampling/importance_sampling_ratio/min": 0.2472153902053833, + "sampling/sampling_logp_difference/max": 1.3974952697753906, + "sampling/sampling_logp_difference/mean": 0.01930988021194935, + "step": 56, + "step_time": 218.33462092967238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12566.0, + "completions/max_terminated_length": 12566.0, + "completions/mean_length": 9388.78125, + "completions/mean_terminated_length": 9388.78125, + "completions/min_length": 6542.0, + "completions/min_terminated_length": 6542.0, + "entropy": 0.2632417492568493, + "epoch": 0.06842737094837935, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3933756649494171, + "learning_rate": 7.76e-07, + "loss": -0.0044, + "num_tokens": 20799727.0, + "reward": 0.655514121055603, + "reward_std": 1.1067219972610474, + "rewards/reward_func/mean": 0.655514121055603, + "rewards/reward_func/std": 1.1067217588424683, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9161028861999512, + "sampling/importance_sampling_ratio/mean": 0.9906187653541565, + "sampling/importance_sampling_ratio/min": 0.03224591165781021, + "sampling/sampling_logp_difference/max": 3.434364080429077, + "sampling/sampling_logp_difference/mean": 0.018842609599232674, + "step": 57, + "step_time": 200.1914525568718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13388.0, + "completions/max_terminated_length": 13388.0, + "completions/mean_length": 9069.125, + "completions/mean_terminated_length": 9069.125, + "completions/min_length": 3500.0, + "completions/min_terminated_length": 3500.0, + "entropy": 0.26544647198170424, + "epoch": 0.06962785114045618, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.34516435861587524, + "learning_rate": 7.72e-07, + "loss": 0.028, + "num_tokens": 21118195.0, + "reward": 1.8895032405853271, + "reward_std": 2.862610101699829, + "rewards/reward_func/mean": 1.8895032405853271, + "rewards/reward_func/std": 2.862610101699829, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9907073378562927, + "sampling/importance_sampling_ratio/min": 0.20175053179264069, + "sampling/sampling_logp_difference/max": 1.6007232666015625, + "sampling/sampling_logp_difference/mean": 0.0191123615950346, + "step": 58, + "step_time": 416.02178934798576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16766.0, + "completions/max_terminated_length": 16766.0, + "completions/mean_length": 9156.90625, + "completions/mean_terminated_length": 9156.90625, + "completions/min_length": 6183.0, + "completions/min_terminated_length": 6183.0, + "entropy": 0.2845827713608742, + "epoch": 0.07082833133253301, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4064350128173828, + "learning_rate": 7.68e-07, + "loss": -0.0212, + "num_tokens": 21436904.0, + "reward": 3.7760183811187744, + "reward_std": 4.1162896156311035, + "rewards/reward_func/mean": 3.778932809829712, + "rewards/reward_func/std": 4.113493919372559, + "rewards/soft_overlong_punishment_reward/mean": -0.0029144287109375, + "rewards/soft_overlong_punishment_reward/std": 0.016486497595906258, + "sampling/importance_sampling_ratio/max": 2.561572790145874, + "sampling/importance_sampling_ratio/mean": 0.9900461435317993, + "sampling/importance_sampling_ratio/min": 0.3691723942756653, + "sampling/sampling_logp_difference/max": 0.996491551399231, + "sampling/sampling_logp_difference/mean": 0.020054107531905174, + "step": 59, + "step_time": 232.3352104361402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15562.0, + "completions/max_terminated_length": 15562.0, + "completions/mean_length": 8621.25, + "completions/mean_terminated_length": 8621.25, + "completions/min_length": 3889.0, + "completions/min_terminated_length": 3889.0, + "entropy": 0.2679576650261879, + "epoch": 0.07202881152460984, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4170191287994385, + "learning_rate": 7.64e-07, + "loss": -0.014, + "num_tokens": 21749208.0, + "reward": 2.277270555496216, + "reward_std": 3.568782329559326, + "rewards/reward_func/mean": 2.277270555496216, + "rewards/reward_func/std": 3.568782329559326, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.22825288772583, + "sampling/importance_sampling_ratio/mean": 0.9907122254371643, + "sampling/importance_sampling_ratio/min": 0.2719807028770447, + "sampling/sampling_logp_difference/max": 1.302024245262146, + "sampling/sampling_logp_difference/mean": 0.019024543464183807, + "step": 60, + "step_time": 241.59232664620504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16602.0, + "completions/max_terminated_length": 16602.0, + "completions/mean_length": 11490.8125, + "completions/mean_terminated_length": 11490.8125, + "completions/min_length": 7465.0, + "completions/min_terminated_length": 7465.0, + "entropy": 0.24971914291381836, + "epoch": 0.07322929171668667, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36966225504875183, + "learning_rate": 7.599999999999999e-07, + "loss": 0.0258, + "num_tokens": 22150906.0, + "reward": 2.009342670440674, + "reward_std": 2.9576900005340576, + "rewards/reward_func/mean": 2.0110058784484863, + "rewards/reward_func/std": 2.956507444381714, + "rewards/soft_overlong_punishment_reward/mean": -0.0016632080078125, + "rewards/soft_overlong_punishment_reward/std": 0.009408525191247463, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9912000894546509, + "sampling/importance_sampling_ratio/min": 0.0796288475394249, + "sampling/sampling_logp_difference/max": 2.530378818511963, + "sampling/sampling_logp_difference/mean": 0.018482467159628868, + "step": 61, + "step_time": 262.58794342912734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16208.0, + "completions/max_terminated_length": 16208.0, + "completions/mean_length": 9912.53125, + "completions/mean_terminated_length": 9912.53125, + "completions/min_length": 3451.0, + "completions/min_terminated_length": 3451.0, + "entropy": 0.2602381184697151, + "epoch": 0.0744297719087635, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45621025562286377, + "learning_rate": 7.559999999999999e-07, + "loss": -0.005, + "num_tokens": 22506499.0, + "reward": 0.5048784017562866, + "reward_std": 0.6571192145347595, + "rewards/reward_func/mean": 0.5048784017562866, + "rewards/reward_func/std": 0.6571192145347595, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.990957498550415, + "sampling/importance_sampling_ratio/min": 0.23703482747077942, + "sampling/sampling_logp_difference/max": 1.4395482540130615, + "sampling/sampling_logp_difference/mean": 0.01870187371969223, + "step": 62, + "step_time": 227.91837301501073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13957.0, + "completions/max_terminated_length": 13957.0, + "completions/mean_length": 10636.0625, + "completions/mean_terminated_length": 10636.0625, + "completions/min_length": 8297.0, + "completions/min_terminated_length": 8297.0, + "entropy": 0.24767167586833239, + "epoch": 0.07563025210084033, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3626573383808136, + "learning_rate": 7.52e-07, + "loss": 0.0097, + "num_tokens": 22889037.0, + "reward": 2.379978656768799, + "reward_std": 2.5725669860839844, + "rewards/reward_func/mean": 2.379978656768799, + "rewards/reward_func/std": 2.5725667476654053, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4255101680755615, + "sampling/importance_sampling_ratio/mean": 0.991312563419342, + "sampling/importance_sampling_ratio/min": 0.31686195731163025, + "sampling/sampling_logp_difference/max": 1.1492891311645508, + "sampling/sampling_logp_difference/mean": 0.017962327226996422, + "step": 63, + "step_time": 220.6857718299143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14398.0, + "completions/max_terminated_length": 14398.0, + "completions/mean_length": 10088.125, + "completions/mean_terminated_length": 10088.125, + "completions/min_length": 6941.0, + "completions/min_terminated_length": 6941.0, + "entropy": 0.25979764945805073, + "epoch": 0.07683073229291716, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3685627281665802, + "learning_rate": 7.48e-07, + "loss": 0.0522, + "num_tokens": 23250569.0, + "reward": 3681.080322265625, + "reward_std": 20817.853515625, + "rewards/reward_func/mean": 3681.080322265625, + "rewards/reward_func/std": 20817.853515625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9908851981163025, + "sampling/importance_sampling_ratio/min": 0.15633250772953033, + "sampling/sampling_logp_difference/max": 1.8557701110839844, + "sampling/sampling_logp_difference/mean": 0.01882169209420681, + "step": 64, + "step_time": 207.90169405727647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15345.0, + "completions/max_terminated_length": 15345.0, + "completions/mean_length": 10208.5625, + "completions/mean_terminated_length": 10208.5625, + "completions/min_length": 4160.0, + "completions/min_terminated_length": 4160.0, + "entropy": 0.25587698817253113, + "epoch": 0.07803121248499399, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37043145298957825, + "learning_rate": 7.44e-07, + "loss": -0.0128, + "num_tokens": 23616123.0, + "reward": 115.96958923339844, + "reward_std": 450.455322265625, + "rewards/reward_func/mean": 115.96958923339844, + "rewards/reward_func/std": 450.455322265625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3009722232818604, + "sampling/importance_sampling_ratio/mean": 0.9911485314369202, + "sampling/importance_sampling_ratio/min": 0.3561326265335083, + "sampling/sampling_logp_difference/max": 1.03245210647583, + "sampling/sampling_logp_difference/mean": 0.018131505697965622, + "step": 65, + "step_time": 247.2077819444239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15775.0, + "completions/max_terminated_length": 15775.0, + "completions/mean_length": 10089.09375, + "completions/mean_terminated_length": 10089.09375, + "completions/min_length": 5130.0, + "completions/min_terminated_length": 5130.0, + "entropy": 0.25899219047278166, + "epoch": 0.07923169267707082, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38848623633384705, + "learning_rate": 7.4e-07, + "loss": -0.0311, + "num_tokens": 23972966.0, + "reward": 2.4493942260742188, + "reward_std": 3.161067247390747, + "rewards/reward_func/mean": 2.4493942260742188, + "rewards/reward_func/std": 3.161067247390747, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.568775177001953, + "sampling/importance_sampling_ratio/mean": 0.9909378290176392, + "sampling/importance_sampling_ratio/min": 0.3397059142589569, + "sampling/sampling_logp_difference/max": 1.0796749591827393, + "sampling/sampling_logp_difference/mean": 0.018584787845611572, + "step": 66, + "step_time": 235.83589306659997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17692.0, + "completions/max_terminated_length": 17692.0, + "completions/mean_length": 11849.34375, + "completions/mean_terminated_length": 11849.34375, + "completions/min_length": 6275.0, + "completions/min_terminated_length": 6275.0, + "entropy": 0.2503598555922508, + "epoch": 0.08043217286914765, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36103734374046326, + "learning_rate": 7.359999999999999e-07, + "loss": 0.0653, + "num_tokens": 24390121.0, + "reward": 0.5631309747695923, + "reward_std": 0.7355284690856934, + "rewards/reward_func/mean": 0.5829674005508423, + "rewards/reward_func/std": 0.7148671746253967, + "rewards/soft_overlong_punishment_reward/mean": -0.01983642578125, + "rewards/soft_overlong_punishment_reward/std": 0.07805701345205307, + "sampling/importance_sampling_ratio/max": 2.5908403396606445, + "sampling/importance_sampling_ratio/mean": 0.9912521243095398, + "sampling/importance_sampling_ratio/min": 0.20392736792564392, + "sampling/sampling_logp_difference/max": 1.5899913311004639, + "sampling/sampling_logp_difference/mean": 0.017768753692507744, + "step": 67, + "step_time": 273.55189195496496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13621.0, + "completions/max_terminated_length": 13621.0, + "completions/mean_length": 10604.21875, + "completions/mean_terminated_length": 10604.21875, + "completions/min_length": 7394.0, + "completions/min_terminated_length": 7394.0, + "entropy": 0.2723994627594948, + "epoch": 0.08163265306122448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40947461128234863, + "learning_rate": 7.319999999999999e-07, + "loss": 0.017, + "num_tokens": 24763560.0, + "reward": 15.798975944519043, + "reward_std": 37.360469818115234, + "rewards/reward_func/mean": 15.798975944519043, + "rewards/reward_func/std": 37.360469818115234, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.846003293991089, + "sampling/importance_sampling_ratio/mean": 0.9903964996337891, + "sampling/importance_sampling_ratio/min": 0.025304390117526054, + "sampling/sampling_logp_difference/max": 3.6767773628234863, + "sampling/sampling_logp_difference/mean": 0.01946277916431427, + "step": 68, + "step_time": 208.66941105970182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12376.0, + "completions/max_terminated_length": 12376.0, + "completions/mean_length": 9659.3125, + "completions/mean_terminated_length": 9659.3125, + "completions/min_length": 7514.0, + "completions/min_terminated_length": 7514.0, + "entropy": 0.2647173348814249, + "epoch": 0.08283313325330131, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3932405710220337, + "learning_rate": 7.28e-07, + "loss": -0.0129, + "num_tokens": 25105250.0, + "reward": 1.1635289192199707, + "reward_std": 1.419382929801941, + "rewards/reward_func/mean": 1.1635289192199707, + "rewards/reward_func/std": 1.4193830490112305, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.187065362930298, + "sampling/importance_sampling_ratio/mean": 0.9907566905021667, + "sampling/importance_sampling_ratio/min": 0.29801252484321594, + "sampling/sampling_logp_difference/max": 1.2106198072433472, + "sampling/sampling_logp_difference/mean": 0.018872233107686043, + "step": 69, + "step_time": 308.8531298188027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15954.0, + "completions/max_terminated_length": 15954.0, + "completions/mean_length": 10261.5625, + "completions/mean_terminated_length": 10261.5625, + "completions/min_length": 6635.0, + "completions/min_terminated_length": 6635.0, + "entropy": 0.25401466619223356, + "epoch": 0.08403361344537816, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.32909703254699707, + "learning_rate": 7.24e-07, + "loss": 0.0189, + "num_tokens": 25472484.0, + "reward": 0.7801880836486816, + "reward_std": 0.7046358585357666, + "rewards/reward_func/mean": 0.7801880836486816, + "rewards/reward_func/std": 0.7046358585357666, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9910746812820435, + "sampling/importance_sampling_ratio/min": 0.15575462579727173, + "sampling/sampling_logp_difference/max": 1.8594733476638794, + "sampling/sampling_logp_difference/mean": 0.018227702006697655, + "step": 70, + "step_time": 458.7579943599412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17229.0, + "completions/max_terminated_length": 17229.0, + "completions/mean_length": 11900.3125, + "completions/mean_terminated_length": 11900.3125, + "completions/min_length": 8069.0, + "completions/min_terminated_length": 8069.0, + "entropy": 0.23596489429473877, + "epoch": 0.08523409363745499, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.28990110754966736, + "learning_rate": 7.2e-07, + "loss": 0.0406, + "num_tokens": 25902358.0, + "reward": 1.1131587028503418, + "reward_std": 2.4644935131073, + "rewards/reward_func/mean": 1.119605541229248, + "rewards/reward_func/std": 2.461198329925537, + "rewards/soft_overlong_punishment_reward/mean": -0.00644683837890625, + "rewards/soft_overlong_punishment_reward/std": 0.03646882623434067, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9916808605194092, + "sampling/importance_sampling_ratio/min": 0.07585223764181137, + "sampling/sampling_logp_difference/max": 2.578968048095703, + "sampling/sampling_logp_difference/mean": 0.017171792685985565, + "step": 71, + "step_time": 256.0422782639507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15647.0, + "completions/max_terminated_length": 15647.0, + "completions/mean_length": 10677.8125, + "completions/mean_terminated_length": 10677.8125, + "completions/min_length": 6661.0, + "completions/min_terminated_length": 6661.0, + "entropy": 0.24026651121675968, + "epoch": 0.08643457382953182, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36302992701530457, + "learning_rate": 7.159999999999999e-07, + "loss": 0.0069, + "num_tokens": 26286872.0, + "reward": 6.745024681091309, + "reward_std": 27.16277313232422, + "rewards/reward_func/mean": 6.745024681091309, + "rewards/reward_func/std": 27.162776947021484, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.476977825164795, + "sampling/importance_sampling_ratio/mean": 0.991825520992279, + "sampling/importance_sampling_ratio/min": 0.006692860741168261, + "sampling/sampling_logp_difference/max": 5.0067138671875, + "sampling/sampling_logp_difference/mean": 0.01745101995766163, + "step": 72, + "step_time": 454.2569543699501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14315.0, + "completions/max_terminated_length": 14315.0, + "completions/mean_length": 6956.90625, + "completions/mean_terminated_length": 6956.90625, + "completions/min_length": 3062.0, + "completions/min_terminated_length": 3062.0, + "entropy": 0.28412431478500366, + "epoch": 0.08763505402160865, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5102128386497498, + "learning_rate": 7.119999999999999e-07, + "loss": 0.0493, + "num_tokens": 26533853.0, + "reward": 1.6690137386322021, + "reward_std": 2.0055153369903564, + "rewards/reward_func/mean": 1.6690137386322021, + "rewards/reward_func/std": 2.0055153369903564, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9905214905738831, + "sampling/importance_sampling_ratio/min": 0.027704769745469093, + "sampling/sampling_logp_difference/max": 3.586150646209717, + "sampling/sampling_logp_difference/mean": 0.019451972097158432, + "step": 73, + "step_time": 203.57053557073232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11038.0, + "completions/max_terminated_length": 11038.0, + "completions/mean_length": 7485.53125, + "completions/mean_terminated_length": 7485.53125, + "completions/min_length": 3940.0, + "completions/min_terminated_length": 3940.0, + "entropy": 0.27887804340571165, + "epoch": 0.08883553421368548, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3595903813838959, + "learning_rate": 7.079999999999999e-07, + "loss": 0.0016, + "num_tokens": 26797934.0, + "reward": 0.582500159740448, + "reward_std": 0.6758586764335632, + "rewards/reward_func/mean": 0.582500159740448, + "rewards/reward_func/std": 0.6758586168289185, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3009722232818604, + "sampling/importance_sampling_ratio/mean": 0.9900974035263062, + "sampling/importance_sampling_ratio/min": 0.3130251169204712, + "sampling/sampling_logp_difference/max": 1.1614718437194824, + "sampling/sampling_logp_difference/mean": 0.01946306601166725, + "step": 74, + "step_time": 394.7669193340698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12980.0, + "completions/max_terminated_length": 12980.0, + "completions/mean_length": 9189.96875, + "completions/mean_terminated_length": 9189.96875, + "completions/min_length": 4670.0, + "completions/min_terminated_length": 4670.0, + "entropy": 0.27417559176683426, + "epoch": 0.09003601440576231, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41591066122055054, + "learning_rate": 7.04e-07, + "loss": 0.0263, + "num_tokens": 27116477.0, + "reward": 1.5546950101852417, + "reward_std": 2.706759452819824, + "rewards/reward_func/mean": 1.5546950101852417, + "rewards/reward_func/std": 2.706759214401245, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9904885292053223, + "sampling/importance_sampling_ratio/min": 0.23027251660823822, + "sampling/sampling_logp_difference/max": 1.468491792678833, + "sampling/sampling_logp_difference/mean": 0.019385498017072678, + "step": 75, + "step_time": 212.57274886220694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15570.0, + "completions/max_terminated_length": 15570.0, + "completions/mean_length": 9589.8125, + "completions/mean_terminated_length": 9589.8125, + "completions/min_length": 3431.0, + "completions/min_terminated_length": 3431.0, + "entropy": 0.24109713081270456, + "epoch": 0.09123649459783914, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39267539978027344, + "learning_rate": 7e-07, + "loss": -0.0101, + "num_tokens": 27469983.0, + "reward": 1.6705238819122314, + "reward_std": 1.578829288482666, + "rewards/reward_func/mean": 1.6705238819122314, + "rewards/reward_func/std": 1.578829288482666, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9916650056838989, + "sampling/importance_sampling_ratio/min": 0.3055680990219116, + "sampling/sampling_logp_difference/max": 1.5075340270996094, + "sampling/sampling_logp_difference/mean": 0.01719709485769272, + "step": 76, + "step_time": 434.341842096881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12644.0, + "completions/max_terminated_length": 12644.0, + "completions/mean_length": 8874.84375, + "completions/mean_terminated_length": 8874.84375, + "completions/min_length": 4780.0, + "completions/min_terminated_length": 4780.0, + "entropy": 0.28068294655531645, + "epoch": 0.09243697478991597, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4375174045562744, + "learning_rate": 6.959999999999999e-07, + "loss": 0.0539, + "num_tokens": 27781946.0, + "reward": 19.96912956237793, + "reward_std": 36.33182144165039, + "rewards/reward_func/mean": 19.96912956237793, + "rewards/reward_func/std": 36.33182144165039, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.198267698287964, + "sampling/importance_sampling_ratio/mean": 0.990230143070221, + "sampling/importance_sampling_ratio/min": 0.1569797843694687, + "sampling/sampling_logp_difference/max": 1.8516383171081543, + "sampling/sampling_logp_difference/mean": 0.01989983581006527, + "step": 77, + "step_time": 399.6355443559587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12663.0, + "completions/max_terminated_length": 12663.0, + "completions/mean_length": 9384.59375, + "completions/mean_terminated_length": 9384.59375, + "completions/min_length": 5366.0, + "completions/min_terminated_length": 5366.0, + "entropy": 0.23270147573202848, + "epoch": 0.0936374549819928, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41008490324020386, + "learning_rate": 6.919999999999999e-07, + "loss": 0.0169, + "num_tokens": 28131637.0, + "reward": 0.34000474214553833, + "reward_std": 0.4872087836265564, + "rewards/reward_func/mean": 0.34000474214553833, + "rewards/reward_func/std": 0.4872088134288788, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9919241666793823, + "sampling/importance_sampling_ratio/min": 0.012431665323674679, + "sampling/sampling_logp_difference/max": 4.387508392333984, + "sampling/sampling_logp_difference/mean": 0.016751548275351524, + "step": 78, + "step_time": 213.20301025058143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13605.0, + "completions/max_terminated_length": 13605.0, + "completions/mean_length": 9480.09375, + "completions/mean_terminated_length": 9480.09375, + "completions/min_length": 6576.0, + "completions/min_terminated_length": 6576.0, + "entropy": 0.2560074757784605, + "epoch": 0.09483793517406963, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4534052312374115, + "learning_rate": 6.879999999999999e-07, + "loss": 0.0027, + "num_tokens": 28473544.0, + "reward": 1.1451480388641357, + "reward_std": 2.02068829536438, + "rewards/reward_func/mean": 1.1451480388641357, + "rewards/reward_func/std": 2.02068829536438, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9910561442375183, + "sampling/importance_sampling_ratio/min": 0.28057095408439636, + "sampling/sampling_logp_difference/max": 1.2709286212921143, + "sampling/sampling_logp_difference/mean": 0.018182622268795967, + "step": 79, + "step_time": 198.80667186016217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14034.0, + "completions/max_terminated_length": 14034.0, + "completions/mean_length": 10575.3125, + "completions/mean_terminated_length": 10575.3125, + "completions/min_length": 7172.0, + "completions/min_terminated_length": 7172.0, + "entropy": 0.25964460615068674, + "epoch": 0.09603841536614646, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4087451696395874, + "learning_rate": 6.84e-07, + "loss": 0.0462, + "num_tokens": 28857042.0, + "reward": 0.8121464252471924, + "reward_std": 1.5952792167663574, + "rewards/reward_func/mean": 0.8121464252471924, + "rewards/reward_func/std": 1.595279335975647, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.889493227005005, + "sampling/importance_sampling_ratio/mean": 0.9910570383071899, + "sampling/importance_sampling_ratio/min": 0.0975920781493187, + "sampling/sampling_logp_difference/max": 2.3269588947296143, + "sampling/sampling_logp_difference/mean": 0.01859731785953045, + "step": 80, + "step_time": 237.49234476010315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17704.0, + "completions/max_terminated_length": 17704.0, + "completions/mean_length": 10183.125, + "completions/mean_terminated_length": 10183.125, + "completions/min_length": 5525.0, + "completions/min_terminated_length": 5525.0, + "entropy": 0.25165214855223894, + "epoch": 0.09723889555822329, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3784518241882324, + "learning_rate": 6.800000000000001e-07, + "loss": -0.0141, + "num_tokens": 29213526.0, + "reward": 8.427579879760742, + "reward_std": 24.21912956237793, + "rewards/reward_func/mean": 8.437650680541992, + "rewards/reward_func/std": 24.21601676940918, + "rewards/soft_overlong_punishment_reward/mean": -0.01007080078125, + "rewards/soft_overlong_punishment_reward/std": 0.0569690503180027, + "sampling/importance_sampling_ratio/max": 2.1546497344970703, + "sampling/importance_sampling_ratio/mean": 0.9912103414535522, + "sampling/importance_sampling_ratio/min": 0.3988558351993561, + "sampling/sampling_logp_difference/max": 0.9191552400588989, + "sampling/sampling_logp_difference/mean": 0.018073324114084244, + "step": 81, + "step_time": 258.5655134237604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12930.0, + "completions/max_terminated_length": 12930.0, + "completions/mean_length": 8081.375, + "completions/mean_terminated_length": 8081.375, + "completions/min_length": 4585.0, + "completions/min_terminated_length": 4585.0, + "entropy": 0.2729193214327097, + "epoch": 0.09843937575030012, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.43891701102256775, + "learning_rate": 6.76e-07, + "loss": 0.0029, + "num_tokens": 29504138.0, + "reward": 2.3440794944763184, + "reward_std": 2.2432892322540283, + "rewards/reward_func/mean": 2.3440794944763184, + "rewards/reward_func/std": 2.2432892322540283, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9903400540351868, + "sampling/importance_sampling_ratio/min": 0.15401123464107513, + "sampling/sampling_logp_difference/max": 1.8707298040390015, + "sampling/sampling_logp_difference/mean": 0.01947890594601631, + "step": 82, + "step_time": 202.33901768375654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14334.0, + "completions/max_terminated_length": 14334.0, + "completions/mean_length": 8477.21875, + "completions/mean_terminated_length": 8477.21875, + "completions/min_length": 2572.0, + "completions/min_terminated_length": 2572.0, + "entropy": 0.26816452480852604, + "epoch": 0.09963985594237695, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.48063552379608154, + "learning_rate": 6.72e-07, + "loss": 0.0714, + "num_tokens": 29807561.0, + "reward": 2.8314692974090576, + "reward_std": 3.5692901611328125, + "rewards/reward_func/mean": 2.8314692974090576, + "rewards/reward_func/std": 3.5692899227142334, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.565842866897583, + "sampling/importance_sampling_ratio/mean": 0.9907183051109314, + "sampling/importance_sampling_ratio/min": 0.19031807780265808, + "sampling/sampling_logp_difference/max": 1.6590585708618164, + "sampling/sampling_logp_difference/mean": 0.018856924027204514, + "step": 83, + "step_time": 217.18472903524525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17204.0, + "completions/max_terminated_length": 17204.0, + "completions/mean_length": 8830.375, + "completions/mean_terminated_length": 8830.375, + "completions/min_length": 4996.0, + "completions/min_terminated_length": 4996.0, + "entropy": 0.27092269994318485, + "epoch": 0.10084033613445378, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41215991973876953, + "learning_rate": 6.68e-07, + "loss": -0.0409, + "num_tokens": 30123829.0, + "reward": 2.2067344188690186, + "reward_std": 3.047388792037964, + "rewards/reward_func/mean": 2.2129905223846436, + "rewards/reward_func/std": 3.0453271865844727, + "rewards/soft_overlong_punishment_reward/mean": -0.006256103515625, + "rewards/soft_overlong_punishment_reward/std": 0.03538986295461655, + "sampling/importance_sampling_ratio/max": 2.052222490310669, + "sampling/importance_sampling_ratio/mean": 0.990820586681366, + "sampling/importance_sampling_ratio/min": 0.3114754259586334, + "sampling/sampling_logp_difference/max": 1.1664347648620605, + "sampling/sampling_logp_difference/mean": 0.0186629518866539, + "step": 84, + "step_time": 211.40592918510083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12600.0, + "completions/max_terminated_length": 12600.0, + "completions/mean_length": 8298.3125, + "completions/mean_terminated_length": 8298.3125, + "completions/min_length": 4949.0, + "completions/min_terminated_length": 4949.0, + "entropy": 0.2829645574092865, + "epoch": 0.10204081632653061, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5601394772529602, + "learning_rate": 6.64e-07, + "loss": 0.0305, + "num_tokens": 30421735.0, + "reward": 3.7544779777526855, + "reward_std": 3.1711204051971436, + "rewards/reward_func/mean": 3.7544779777526855, + "rewards/reward_func/std": 3.1711206436157227, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9901652932167053, + "sampling/importance_sampling_ratio/min": 0.12367851287126541, + "sampling/sampling_logp_difference/max": 2.0900697708129883, + "sampling/sampling_logp_difference/mean": 0.020197300240397453, + "step": 85, + "step_time": 266.99491723254323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 15285.0, + "completions/mean_length": 10550.75, + "completions/mean_terminated_length": 10230.451171875, + "completions/min_length": 5820.0, + "completions/min_terminated_length": 5820.0, + "entropy": 0.24555738922208548, + "epoch": 0.10324129651860744, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.314035564661026, + "learning_rate": 6.6e-07, + "loss": -0.0973, + "num_tokens": 30792879.0, + "reward": 1.717525601387024, + "reward_std": 3.4412992000579834, + "rewards/reward_func/mean": 1.805187702178955, + "rewards/reward_func/std": 3.4616737365722656, + "rewards/soft_overlong_punishment_reward/mean": -0.03125, + "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9913591742515564, + "sampling/importance_sampling_ratio/min": 0.16628147661685944, + "sampling/sampling_logp_difference/max": 1.7940733432769775, + "sampling/sampling_logp_difference/mean": 0.017975792288780212, + "step": 86, + "step_time": 266.66299685207196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13159.0, + "completions/max_terminated_length": 13159.0, + "completions/mean_length": 9776.40625, + "completions/mean_terminated_length": 9776.40625, + "completions/min_length": 7210.0, + "completions/min_terminated_length": 7210.0, + "entropy": 0.24749416019767523, + "epoch": 0.10444177671068428, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37279677391052246, + "learning_rate": 6.56e-07, + "loss": -0.0527, + "num_tokens": 31140836.0, + "reward": 2.521646738052368, + "reward_std": 3.81414794921875, + "rewards/reward_func/mean": 2.521646738052368, + "rewards/reward_func/std": 3.81414794921875, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4072625637054443, + "sampling/importance_sampling_ratio/mean": 0.9911715984344482, + "sampling/importance_sampling_ratio/min": 0.19457262754440308, + "sampling/sampling_logp_difference/max": 1.6369497776031494, + "sampling/sampling_logp_difference/mean": 0.017878280952572823, + "step": 87, + "step_time": 313.04910568264313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16942.0, + "completions/max_terminated_length": 16942.0, + "completions/mean_length": 9632.09375, + "completions/mean_terminated_length": 9632.09375, + "completions/min_length": 3954.0, + "completions/min_terminated_length": 3954.0, + "entropy": 0.28111381735652685, + "epoch": 0.1056422569027611, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41607415676116943, + "learning_rate": 6.52e-07, + "loss": 0.0437, + "num_tokens": 31474263.0, + "reward": 1.1277384757995605, + "reward_std": 1.8439369201660156, + "rewards/reward_func/mean": 1.131995677947998, + "rewards/reward_func/std": 1.8411990404129028, + "rewards/soft_overlong_punishment_reward/mean": -0.0042572021484375, + "rewards/soft_overlong_punishment_reward/std": 0.024082370102405548, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9902504682540894, + "sampling/importance_sampling_ratio/min": 0.01312999241054058, + "sampling/sampling_logp_difference/max": 4.332856178283691, + "sampling/sampling_logp_difference/mean": 0.01968482695519924, + "step": 88, + "step_time": 231.826713241986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16728.0, + "completions/max_terminated_length": 16728.0, + "completions/mean_length": 10530.4375, + "completions/mean_terminated_length": 10530.4375, + "completions/min_length": 6271.0, + "completions/min_terminated_length": 6271.0, + "entropy": 0.25707424618303776, + "epoch": 0.10684273709483794, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38471195101737976, + "learning_rate": 6.48e-07, + "loss": 0.0606, + "num_tokens": 31852261.0, + "reward": 1.277016282081604, + "reward_std": 2.039738893508911, + "rewards/reward_func/mean": 1.279640793800354, + "rewards/reward_func/std": 2.038050889968872, + "rewards/soft_overlong_punishment_reward/mean": -0.00262451171875, + "rewards/soft_overlong_punishment_reward/std": 0.014846480451524258, + "sampling/importance_sampling_ratio/max": 2.706266164779663, + "sampling/importance_sampling_ratio/mean": 0.9911405444145203, + "sampling/importance_sampling_ratio/min": 0.22103802859783173, + "sampling/sampling_logp_difference/max": 1.5094205141067505, + "sampling/sampling_logp_difference/mean": 0.0181552916765213, + "step": 89, + "step_time": 452.72294589015655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15561.0, + "completions/max_terminated_length": 15561.0, + "completions/mean_length": 10859.5, + "completions/mean_terminated_length": 10859.5, + "completions/min_length": 4952.0, + "completions/min_terminated_length": 4952.0, + "entropy": 0.24442417547106743, + "epoch": 0.10804321728691477, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37132179737091064, + "learning_rate": 6.44e-07, + "loss": 0.025, + "num_tokens": 32237653.0, + "reward": 0.5021224617958069, + "reward_std": 0.7325555682182312, + "rewards/reward_func/mean": 0.5021224617958069, + "rewards/reward_func/std": 0.7325555086135864, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3009634017944336, + "sampling/importance_sampling_ratio/mean": 0.9916141033172607, + "sampling/importance_sampling_ratio/min": 0.0029834972228854895, + "sampling/sampling_logp_difference/max": 5.814659118652344, + "sampling/sampling_logp_difference/mean": 0.017731234431266785, + "step": 90, + "step_time": 248.43443835526705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14837.0, + "completions/max_terminated_length": 14837.0, + "completions/mean_length": 10019.59375, + "completions/mean_terminated_length": 10019.59375, + "completions/min_length": 6964.0, + "completions/min_terminated_length": 6964.0, + "entropy": 0.279639994725585, + "epoch": 0.1092436974789916, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3926539421081543, + "learning_rate": 6.4e-07, + "loss": 0.0568, + "num_tokens": 32596080.0, + "reward": 2894.755615234375, + "reward_std": 7884.30615234375, + "rewards/reward_func/mean": 2894.755615234375, + "rewards/reward_func/std": 7884.30615234375, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9901866316795349, + "sampling/importance_sampling_ratio/min": 0.2964800298213959, + "sampling/sampling_logp_difference/max": 1.3845736980438232, + "sampling/sampling_logp_difference/mean": 0.019792938604950905, + "step": 91, + "step_time": 230.6299822588917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12330.0, + "completions/max_terminated_length": 12330.0, + "completions/mean_length": 9052.78125, + "completions/mean_terminated_length": 9052.78125, + "completions/min_length": 5906.0, + "completions/min_terminated_length": 5906.0, + "entropy": 0.31160302832722664, + "epoch": 0.11044417767106843, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41976141929626465, + "learning_rate": 6.36e-07, + "loss": -0.0106, + "num_tokens": 32908137.0, + "reward": 1.0270764827728271, + "reward_std": 1.5837911367416382, + "rewards/reward_func/mean": 1.0270764827728271, + "rewards/reward_func/std": 1.5837911367416382, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9889901280403137, + "sampling/importance_sampling_ratio/min": 0.251454621553421, + "sampling/sampling_logp_difference/max": 1.3804928064346313, + "sampling/sampling_logp_difference/mean": 0.02152213454246521, + "step": 92, + "step_time": 209.22184566501528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13164.0, + "completions/max_terminated_length": 13164.0, + "completions/mean_length": 7722.125, + "completions/mean_terminated_length": 7722.125, + "completions/min_length": 3235.0, + "completions/min_terminated_length": 3235.0, + "entropy": 0.27631482668220997, + "epoch": 0.11164465786314526, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.43816474080085754, + "learning_rate": 6.319999999999999e-07, + "loss": -0.0276, + "num_tokens": 33179125.0, + "reward": 3.121471881866455, + "reward_std": 3.0470423698425293, + "rewards/reward_func/mean": 3.121471881866455, + "rewards/reward_func/std": 3.0470426082611084, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2446866035461426, + "sampling/importance_sampling_ratio/mean": 0.990494430065155, + "sampling/importance_sampling_ratio/min": 0.3229256570339203, + "sampling/sampling_logp_difference/max": 1.1303331851959229, + "sampling/sampling_logp_difference/mean": 0.0194493867456913, + "step": 93, + "step_time": 403.52912656613626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 9834.0, + "completions/max_terminated_length": 9834.0, + "completions/mean_length": 6429.0, + "completions/mean_terminated_length": 6429.0, + "completions/min_length": 3298.0, + "completions/min_terminated_length": 3298.0, + "entropy": 0.3079283908009529, + "epoch": 0.11284513805522209, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.512400209903717, + "learning_rate": 6.28e-07, + "loss": -0.0118, + "num_tokens": 33409485.0, + "reward": 4.05039119720459, + "reward_std": 6.237717628479004, + "rewards/reward_func/mean": 4.05039119720459, + "rewards/reward_func/std": 6.237717151641846, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9068455696105957, + "sampling/importance_sampling_ratio/mean": 0.9891414046287537, + "sampling/importance_sampling_ratio/min": 0.28006723523139954, + "sampling/sampling_logp_difference/max": 1.2727255821228027, + "sampling/sampling_logp_difference/mean": 0.021337833255529404, + "step": 94, + "step_time": 150.58229626482353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 13641.0, + "completions/mean_length": 8809.875, + "completions/mean_terminated_length": 8433.4189453125, + "completions/min_length": 2494.0, + "completions/min_terminated_length": 2494.0, + "entropy": 0.28533732518553734, + "epoch": 0.11404561824729892, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34757664799690247, + "learning_rate": 6.24e-07, + "loss": -0.1059, + "num_tokens": 33722393.0, + "reward": 2.1445155143737793, + "reward_std": 2.832806348800659, + "rewards/reward_func/mean": 2.2459514141082764, + "rewards/reward_func/std": 2.8199384212493896, + "rewards/soft_overlong_punishment_reward/mean": -0.03125, + "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9901010990142822, + "sampling/importance_sampling_ratio/min": 0.0004452175635378808, + "sampling/sampling_logp_difference/max": 7.716947555541992, + "sampling/sampling_logp_difference/mean": 0.01991894841194153, + "step": 95, + "step_time": 253.43176098098047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15697.0, + "completions/max_terminated_length": 15697.0, + "completions/mean_length": 10009.0625, + "completions/mean_terminated_length": 10009.0625, + "completions/min_length": 5990.0, + "completions/min_terminated_length": 5990.0, + "entropy": 0.24590188823640347, + "epoch": 0.11524609843937575, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.316346138715744, + "learning_rate": 6.2e-07, + "loss": -0.0135, + "num_tokens": 34078643.0, + "reward": 0.39841264486312866, + "reward_std": 0.5363882184028625, + "rewards/reward_func/mean": 0.39841264486312866, + "rewards/reward_func/std": 0.5363882184028625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.871351718902588, + "sampling/importance_sampling_ratio/mean": 0.9914729595184326, + "sampling/importance_sampling_ratio/min": 0.09377329051494598, + "sampling/sampling_logp_difference/max": 2.366875171661377, + "sampling/sampling_logp_difference/mean": 0.017338084056973457, + "step": 96, + "step_time": 435.25480223365594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14781.0, + "completions/max_terminated_length": 14781.0, + "completions/mean_length": 10244.75, + "completions/mean_terminated_length": 10244.75, + "completions/min_length": 5004.0, + "completions/min_terminated_length": 5004.0, + "entropy": 0.3078562542796135, + "epoch": 0.11644657863145258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4103779196739197, + "learning_rate": 6.16e-07, + "loss": 0.038, + "num_tokens": 34434571.0, + "reward": 0.8614925146102905, + "reward_std": 0.5891639590263367, + "rewards/reward_func/mean": 0.8614925146102905, + "rewards/reward_func/std": 0.5891639590263367, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2111990451812744, + "sampling/importance_sampling_ratio/mean": 0.9891133308410645, + "sampling/importance_sampling_ratio/min": 0.1545713096857071, + "sampling/sampling_logp_difference/max": 1.8670997619628906, + "sampling/sampling_logp_difference/mean": 0.021383928135037422, + "step": 97, + "step_time": 212.63684044289403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15325.0, + "completions/max_terminated_length": 15325.0, + "completions/mean_length": 11469.84375, + "completions/mean_terminated_length": 11469.84375, + "completions/min_length": 6985.0, + "completions/min_terminated_length": 6985.0, + "entropy": 0.26262489426881075, + "epoch": 0.11764705882352941, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37121981382369995, + "learning_rate": 6.119999999999999e-07, + "loss": 0.0254, + "num_tokens": 34846182.0, + "reward": 1.391387701034546, + "reward_std": 2.258481025695801, + "rewards/reward_func/mean": 1.391387701034546, + "rewards/reward_func/std": 2.258481025695801, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9727016687393188, + "sampling/importance_sampling_ratio/mean": 0.9908169507980347, + "sampling/importance_sampling_ratio/min": 0.24390530586242676, + "sampling/sampling_logp_difference/max": 1.4109752178192139, + "sampling/sampling_logp_difference/mean": 0.018506620079278946, + "step": 98, + "step_time": 251.3301967199659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12299.0, + "completions/max_terminated_length": 12299.0, + "completions/mean_length": 8463.15625, + "completions/mean_terminated_length": 8463.15625, + "completions/min_length": 3478.0, + "completions/min_terminated_length": 3478.0, + "entropy": 0.273245669901371, + "epoch": 0.11884753901560624, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.365919291973114, + "learning_rate": 6.079999999999999e-07, + "loss": 0.0383, + "num_tokens": 35145083.0, + "reward": 3.128838539123535, + "reward_std": 3.8530807495117188, + "rewards/reward_func/mean": 3.128838539123535, + "rewards/reward_func/std": 3.853080987930298, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.477442979812622, + "sampling/importance_sampling_ratio/mean": 0.9905266761779785, + "sampling/importance_sampling_ratio/min": 0.38133692741394043, + "sampling/sampling_logp_difference/max": 0.9640719890594482, + "sampling/sampling_logp_difference/mean": 0.0193619467318058, + "step": 99, + "step_time": 214.26986349211074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14368.0, + "completions/max_terminated_length": 14368.0, + "completions/mean_length": 8256.4375, + "completions/mean_terminated_length": 8256.4375, + "completions/min_length": 4534.0, + "completions/min_terminated_length": 4534.0, + "entropy": 0.3049591239541769, + "epoch": 0.12004801920768307, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.38011953234672546, + "learning_rate": 6.04e-07, + "loss": -0.0081, + "num_tokens": 35432593.0, + "reward": 2.3642165660858154, + "reward_std": 2.706695556640625, + "rewards/reward_func/mean": 2.3642165660858154, + "rewards/reward_func/std": 2.706695556640625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9068528413772583, + "sampling/importance_sampling_ratio/mean": 0.9893739223480225, + "sampling/importance_sampling_ratio/min": 0.1614706665277481, + "sampling/sampling_logp_difference/max": 1.8234318494796753, + "sampling/sampling_logp_difference/mean": 0.02130398340523243, + "step": 100, + "step_time": 184.41112436982803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17413.0, + "completions/max_terminated_length": 17413.0, + "completions/mean_length": 11627.0625, + "completions/mean_terminated_length": 11627.0625, + "completions/min_length": 6505.0, + "completions/min_terminated_length": 6505.0, + "entropy": 0.24142704904079437, + "epoch": 0.1212484993997599, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36239883303642273, + "learning_rate": 6e-07, + "loss": 0.0495, + "num_tokens": 35847699.0, + "reward": 0.8143461346626282, + "reward_std": 0.6679160594940186, + "rewards/reward_func/mean": 0.8221967816352844, + "rewards/reward_func/std": 0.6563639044761658, + "rewards/soft_overlong_punishment_reward/mean": -0.00785064697265625, + "rewards/soft_overlong_punishment_reward/std": 0.044409964233636856, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9916003346443176, + "sampling/importance_sampling_ratio/min": 0.05457422509789467, + "sampling/sampling_logp_difference/max": 2.908193588256836, + "sampling/sampling_logp_difference/mean": 0.01743597723543644, + "step": 101, + "step_time": 247.9607744079549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12313.0, + "completions/max_terminated_length": 12313.0, + "completions/mean_length": 9030.0625, + "completions/mean_terminated_length": 9030.0625, + "completions/min_length": 6142.0, + "completions/min_terminated_length": 6142.0, + "entropy": 0.2866959013044834, + "epoch": 0.12244897959183673, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4205459654331207, + "learning_rate": 5.96e-07, + "loss": 0.0247, + "num_tokens": 36166773.0, + "reward": 1.333907127380371, + "reward_std": 1.1282483339309692, + "rewards/reward_func/mean": 1.333907127380371, + "rewards/reward_func/std": 1.1282483339309692, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.730395555496216, + "sampling/importance_sampling_ratio/mean": 0.9900311231613159, + "sampling/importance_sampling_ratio/min": 0.05878521502017975, + "sampling/sampling_logp_difference/max": 2.83386492729187, + "sampling/sampling_logp_difference/mean": 0.020093757659196854, + "step": 102, + "step_time": 215.10454945149831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12475.0, + "completions/max_terminated_length": 12475.0, + "completions/mean_length": 8976.34375, + "completions/mean_terminated_length": 8976.34375, + "completions/min_length": 3906.0, + "completions/min_terminated_length": 3906.0, + "entropy": 0.24582395143806934, + "epoch": 0.12364945978391356, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3737013339996338, + "learning_rate": 5.919999999999999e-07, + "loss": 0.0012, + "num_tokens": 36486264.0, + "reward": 0.4444224536418915, + "reward_std": 0.6303268671035767, + "rewards/reward_func/mean": 0.4444224536418915, + "rewards/reward_func/std": 0.6303268074989319, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9914492964744568, + "sampling/importance_sampling_ratio/min": 0.18756607174873352, + "sampling/sampling_logp_difference/max": 1.673624038696289, + "sampling/sampling_logp_difference/mean": 0.01754648983478546, + "step": 103, + "step_time": 206.92123383819126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13655.0, + "completions/max_terminated_length": 13655.0, + "completions/mean_length": 9150.15625, + "completions/mean_terminated_length": 9150.15625, + "completions/min_length": 4648.0, + "completions/min_terminated_length": 4648.0, + "entropy": 0.26344494987279177, + "epoch": 0.12484993997599039, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.34451353549957275, + "learning_rate": 5.879999999999999e-07, + "loss": -0.0295, + "num_tokens": 36814093.0, + "reward": 5.4827399253845215, + "reward_std": 10.697521209716797, + "rewards/reward_func/mean": 5.4827399253845215, + "rewards/reward_func/std": 10.697521209716797, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3717105388641357, + "sampling/importance_sampling_ratio/mean": 0.9908387064933777, + "sampling/importance_sampling_ratio/min": 0.23612551391124725, + "sampling/sampling_logp_difference/max": 1.4433917999267578, + "sampling/sampling_logp_difference/mean": 0.018703076988458633, + "step": 104, + "step_time": 213.1645632191794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14144.0, + "completions/max_terminated_length": 14144.0, + "completions/mean_length": 9819.1875, + "completions/mean_terminated_length": 9819.1875, + "completions/min_length": 5625.0, + "completions/min_terminated_length": 5625.0, + "entropy": 0.24409340787678957, + "epoch": 0.12605042016806722, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.33012041449546814, + "learning_rate": 5.839999999999999e-07, + "loss": 0.029, + "num_tokens": 37167963.0, + "reward": 1.2769612073898315, + "reward_std": 2.2972137928009033, + "rewards/reward_func/mean": 1.2769612073898315, + "rewards/reward_func/std": 2.2972140312194824, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3744382858276367, + "sampling/importance_sampling_ratio/mean": 0.9916211366653442, + "sampling/importance_sampling_ratio/min": 0.18144047260284424, + "sampling/sampling_logp_difference/max": 1.7068276405334473, + "sampling/sampling_logp_difference/mean": 0.017610425129532814, + "step": 105, + "step_time": 233.00164715701248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19045.0, + "completions/max_terminated_length": 19045.0, + "completions/mean_length": 10475.28125, + "completions/mean_terminated_length": 10475.28125, + "completions/min_length": 6236.0, + "completions/min_terminated_length": 6236.0, + "entropy": 0.2810442354530096, + "epoch": 0.12725090036014405, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4047479033470154, + "learning_rate": 5.8e-07, + "loss": 0.0844, + "num_tokens": 37540988.0, + "reward": 1.971191644668579, + "reward_std": 3.9106719493865967, + "rewards/reward_func/mean": 1.9987413883209229, + "rewards/reward_func/std": 3.894240140914917, + "rewards/soft_overlong_punishment_reward/mean": -0.02754974365234375, + "rewards/soft_overlong_punishment_reward/std": 0.12069173902273178, + "sampling/importance_sampling_ratio/max": 2.3556199073791504, + "sampling/importance_sampling_ratio/mean": 0.990192711353302, + "sampling/importance_sampling_ratio/min": 0.1058955043554306, + "sampling/sampling_logp_difference/max": 2.245302438735962, + "sampling/sampling_logp_difference/mean": 0.019729377701878548, + "step": 106, + "step_time": 259.3078554049134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11035.0, + "completions/max_terminated_length": 11035.0, + "completions/mean_length": 8085.09375, + "completions/mean_terminated_length": 8085.09375, + "completions/min_length": 5843.0, + "completions/min_terminated_length": 5843.0, + "entropy": 0.28292676340788603, + "epoch": 0.12845138055222088, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44497084617614746, + "learning_rate": 5.76e-07, + "loss": -0.0027, + "num_tokens": 37824127.0, + "reward": 5.807962417602539, + "reward_std": 3.6478431224823, + "rewards/reward_func/mean": 5.807962417602539, + "rewards/reward_func/std": 3.6478431224823, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9902195930480957, + "sampling/importance_sampling_ratio/min": 0.2602680027484894, + "sampling/sampling_logp_difference/max": 1.6943206787109375, + "sampling/sampling_logp_difference/mean": 0.01987529546022415, + "step": 107, + "step_time": 164.33568983571604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16656.0, + "completions/max_terminated_length": 16656.0, + "completions/mean_length": 10496.1875, + "completions/mean_terminated_length": 10496.1875, + "completions/min_length": 6152.0, + "completions/min_terminated_length": 6152.0, + "entropy": 0.31788551434874535, + "epoch": 0.12965186074429771, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.36823001503944397, + "learning_rate": 5.719999999999999e-07, + "loss": 0.0434, + "num_tokens": 38184413.0, + "reward": 0.822684109210968, + "reward_std": 0.7153370380401611, + "rewards/reward_func/mean": 0.824759304523468, + "rewards/reward_func/std": 0.7129164934158325, + "rewards/soft_overlong_punishment_reward/mean": -0.0020751953125, + "rewards/soft_overlong_punishment_reward/std": 0.011739077046513557, + "sampling/importance_sampling_ratio/max": 2.7742605209350586, + "sampling/importance_sampling_ratio/mean": 0.9888496994972229, + "sampling/importance_sampling_ratio/min": 0.3614434599876404, + "sampling/sampling_logp_difference/max": 1.0203841924667358, + "sampling/sampling_logp_difference/mean": 0.0220322422683239, + "step": 108, + "step_time": 238.20601075387094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14831.0, + "completions/max_terminated_length": 14831.0, + "completions/mean_length": 10238.46875, + "completions/mean_terminated_length": 10238.46875, + "completions/min_length": 6359.0, + "completions/min_terminated_length": 6359.0, + "entropy": 0.28053270652890205, + "epoch": 0.13085234093637454, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39827480912208557, + "learning_rate": 5.679999999999999e-07, + "loss": -0.0661, + "num_tokens": 38553132.0, + "reward": 0.5400049686431885, + "reward_std": 0.8026726245880127, + "rewards/reward_func/mean": 0.5400049686431885, + "rewards/reward_func/std": 0.8026726841926575, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3182525634765625, + "sampling/importance_sampling_ratio/mean": 0.9902611374855042, + "sampling/importance_sampling_ratio/min": 0.3386656939983368, + "sampling/sampling_logp_difference/max": 1.0827418565750122, + "sampling/sampling_logp_difference/mean": 0.01999467983841896, + "step": 109, + "step_time": 251.36417699581943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12513.0, + "completions/max_terminated_length": 12513.0, + "completions/mean_length": 9081.59375, + "completions/mean_terminated_length": 9081.59375, + "completions/min_length": 4968.0, + "completions/min_terminated_length": 4968.0, + "entropy": 0.2736341943964362, + "epoch": 0.13205282112845138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4176849126815796, + "learning_rate": 5.639999999999999e-07, + "loss": 0.0632, + "num_tokens": 38883943.0, + "reward": 1.397516131401062, + "reward_std": 1.4397801160812378, + "rewards/reward_func/mean": 1.397516131401062, + "rewards/reward_func/std": 1.4397801160812378, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.864140510559082, + "sampling/importance_sampling_ratio/mean": 0.9903908371925354, + "sampling/importance_sampling_ratio/min": 0.30289867520332336, + "sampling/sampling_logp_difference/max": 1.194356918334961, + "sampling/sampling_logp_difference/mean": 0.019294166937470436, + "step": 110, + "step_time": 432.76785584609024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15302.0, + "completions/max_terminated_length": 15302.0, + "completions/mean_length": 10490.46875, + "completions/mean_terminated_length": 10490.46875, + "completions/min_length": 7700.0, + "completions/min_terminated_length": 7700.0, + "entropy": 0.2960231024771929, + "epoch": 0.1332533013205282, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4143314063549042, + "learning_rate": 5.6e-07, + "loss": 0.0137, + "num_tokens": 39249678.0, + "reward": 1.0926156044006348, + "reward_std": 0.6936445832252502, + "rewards/reward_func/mean": 1.0926156044006348, + "rewards/reward_func/std": 0.6936445236206055, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.972447633743286, + "sampling/importance_sampling_ratio/mean": 0.9898686408996582, + "sampling/importance_sampling_ratio/min": 0.15995651483535767, + "sampling/sampling_logp_difference/max": 1.8328533172607422, + "sampling/sampling_logp_difference/mean": 0.02054767683148384, + "step": 111, + "step_time": 220.7248961061705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17000.0, + "completions/max_terminated_length": 17000.0, + "completions/mean_length": 9145.03125, + "completions/mean_terminated_length": 9145.03125, + "completions/min_length": 3641.0, + "completions/min_terminated_length": 3641.0, + "entropy": 0.2596113607287407, + "epoch": 0.13445378151260504, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42476722598075867, + "learning_rate": 5.560000000000001e-07, + "loss": 0.0171, + "num_tokens": 39581495.0, + "reward": 0.21374335885047913, + "reward_std": 0.40230268239974976, + "rewards/reward_func/mean": 0.21844306588172913, + "rewards/reward_func/std": 0.39877456426620483, + "rewards/soft_overlong_punishment_reward/mean": -0.00469970703125, + "rewards/soft_overlong_punishment_reward/std": 0.02658555842936039, + "sampling/importance_sampling_ratio/max": 2.445462942123413, + "sampling/importance_sampling_ratio/mean": 0.9910917282104492, + "sampling/importance_sampling_ratio/min": 0.0019266947638243437, + "sampling/sampling_logp_difference/max": 6.251949310302734, + "sampling/sampling_logp_difference/mean": 0.018219787627458572, + "step": 112, + "step_time": 280.31022595707327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13890.0, + "completions/max_terminated_length": 13890.0, + "completions/mean_length": 10680.34375, + "completions/mean_terminated_length": 10680.34375, + "completions/min_length": 6448.0, + "completions/min_terminated_length": 6448.0, + "entropy": 0.28790590912103653, + "epoch": 0.13565426170468187, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3604806065559387, + "learning_rate": 5.520000000000001e-07, + "loss": -0.0006, + "num_tokens": 39955530.0, + "reward": 0.5530512928962708, + "reward_std": 0.9826109409332275, + "rewards/reward_func/mean": 0.5530512928962708, + "rewards/reward_func/std": 0.9826108813285828, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.565842866897583, + "sampling/importance_sampling_ratio/mean": 0.9898175597190857, + "sampling/importance_sampling_ratio/min": 0.2070624679327011, + "sampling/sampling_logp_difference/max": 1.5747346878051758, + "sampling/sampling_logp_difference/mean": 0.02044212445616722, + "step": 113, + "step_time": 420.17655016179197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13642.0, + "completions/max_terminated_length": 13642.0, + "completions/mean_length": 9351.1875, + "completions/mean_terminated_length": 9351.1875, + "completions/min_length": 5824.0, + "completions/min_terminated_length": 5824.0, + "entropy": 0.2932744771242142, + "epoch": 0.1368547418967587, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4188636839389801, + "learning_rate": 5.48e-07, + "loss": 0.004, + "num_tokens": 40283608.0, + "reward": 1.3525290489196777, + "reward_std": 1.7022446393966675, + "rewards/reward_func/mean": 1.3525290489196777, + "rewards/reward_func/std": 1.702244520187378, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989858090877533, + "sampling/importance_sampling_ratio/min": 0.2756859362125397, + "sampling/sampling_logp_difference/max": 2.190220355987549, + "sampling/sampling_logp_difference/mean": 0.020323438569903374, + "step": 114, + "step_time": 226.98730401217472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17342.0, + "completions/max_terminated_length": 17342.0, + "completions/mean_length": 8931.0, + "completions/mean_terminated_length": 8931.0, + "completions/min_length": 3315.0, + "completions/min_terminated_length": 3315.0, + "entropy": 0.24458013847470284, + "epoch": 0.13805522208883553, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3886120617389679, + "learning_rate": 5.44e-07, + "loss": -0.0341, + "num_tokens": 40610928.0, + "reward": 0.9550597667694092, + "reward_std": 1.4101365804672241, + "rewards/reward_func/mean": 0.9623687267303467, + "rewards/reward_func/std": 1.4116019010543823, + "rewards/soft_overlong_punishment_reward/mean": -0.0073089599609375, + "rewards/soft_overlong_punishment_reward/std": 0.04134571924805641, + "sampling/importance_sampling_ratio/max": 2.6611251831054688, + "sampling/importance_sampling_ratio/mean": 0.9916718006134033, + "sampling/importance_sampling_ratio/min": 0.16927804052829742, + "sampling/sampling_logp_difference/max": 1.7762126922607422, + "sampling/sampling_logp_difference/mean": 0.017050782218575478, + "step": 115, + "step_time": 229.30203108501155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15122.0, + "completions/max_terminated_length": 15122.0, + "completions/mean_length": 10591.75, + "completions/mean_terminated_length": 10591.75, + "completions/min_length": 7260.0, + "completions/min_terminated_length": 7260.0, + "entropy": 0.2582763684913516, + "epoch": 0.13925570228091236, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3612559139728546, + "learning_rate": 5.4e-07, + "loss": 0.0453, + "num_tokens": 40996336.0, + "reward": 2.7374861240386963, + "reward_std": 3.0827701091766357, + "rewards/reward_func/mean": 2.7374861240386963, + "rewards/reward_func/std": 3.0827698707580566, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.93190598487854, + "sampling/importance_sampling_ratio/mean": 0.9909803867340088, + "sampling/importance_sampling_ratio/min": 0.031009899452328682, + "sampling/sampling_logp_difference/max": 3.4734487533569336, + "sampling/sampling_logp_difference/mean": 0.018270010128617287, + "step": 116, + "step_time": 227.7563733382849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13868.0, + "completions/max_terminated_length": 13868.0, + "completions/mean_length": 9498.53125, + "completions/mean_terminated_length": 9498.53125, + "completions/min_length": 5313.0, + "completions/min_terminated_length": 5313.0, + "entropy": 0.2784241568297148, + "epoch": 0.1404561824729892, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5930799841880798, + "learning_rate": 5.36e-07, + "loss": -0.033, + "num_tokens": 41336345.0, + "reward": 0.9384326338768005, + "reward_std": 1.186398983001709, + "rewards/reward_func/mean": 0.9384326338768005, + "rewards/reward_func/std": 1.1863987445831299, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.455018997192383, + "sampling/importance_sampling_ratio/mean": 0.9903002977371216, + "sampling/importance_sampling_ratio/min": 0.015267138369381428, + "sampling/sampling_logp_difference/max": 4.1820526123046875, + "sampling/sampling_logp_difference/mean": 0.019709795713424683, + "step": 117, + "step_time": 243.17280280101113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14484.0, + "completions/max_terminated_length": 14484.0, + "completions/mean_length": 9971.8125, + "completions/mean_terminated_length": 9971.8125, + "completions/min_length": 7100.0, + "completions/min_terminated_length": 7100.0, + "entropy": 0.29973538406193256, + "epoch": 0.14165666266506602, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5883257985115051, + "learning_rate": 5.32e-07, + "loss": 0.0281, + "num_tokens": 41688307.0, + "reward": 0.6539914608001709, + "reward_std": 0.7319720983505249, + "rewards/reward_func/mean": 0.6539914608001709, + "rewards/reward_func/std": 0.7319720387458801, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.818315029144287, + "sampling/importance_sampling_ratio/mean": 0.9895184636116028, + "sampling/importance_sampling_ratio/min": 0.4023731052875519, + "sampling/sampling_logp_difference/max": 1.0361392498016357, + "sampling/sampling_logp_difference/mean": 0.021014517173171043, + "step": 118, + "step_time": 231.1251015268499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19530.0, + "completions/max_terminated_length": 19530.0, + "completions/mean_length": 11520.625, + "completions/mean_terminated_length": 11520.625, + "completions/min_length": 3825.0, + "completions/min_terminated_length": 3825.0, + "entropy": 0.24881694186478853, + "epoch": 0.14285714285714285, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3857250213623047, + "learning_rate": 5.28e-07, + "loss": -0.0142, + "num_tokens": 42102007.0, + "reward": 1.7721220254898071, + "reward_std": 2.1013307571411133, + "rewards/reward_func/mean": 1.8261228799819946, + "rewards/reward_func/std": 2.050947427749634, + "rewards/soft_overlong_punishment_reward/mean": -0.0540008544921875, + "rewards/soft_overlong_punishment_reward/std": 0.18218861520290375, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9912911653518677, + "sampling/importance_sampling_ratio/min": 0.07585831731557846, + "sampling/sampling_logp_difference/max": 2.578887939453125, + "sampling/sampling_logp_difference/mean": 0.017922591418027878, + "step": 119, + "step_time": 291.4575341643067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15462.0, + "completions/max_terminated_length": 15462.0, + "completions/mean_length": 8598.9375, + "completions/mean_terminated_length": 8598.9375, + "completions/min_length": 3214.0, + "completions/min_terminated_length": 3214.0, + "entropy": 0.25942704640328884, + "epoch": 0.14405762304921968, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.37554553151130676, + "learning_rate": 5.24e-07, + "loss": 0.0008, + "num_tokens": 42406077.0, + "reward": 1.3100762367248535, + "reward_std": 1.914520263671875, + "rewards/reward_func/mean": 1.3100762367248535, + "rewards/reward_func/std": 1.914520263671875, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4972586631774902, + "sampling/importance_sampling_ratio/mean": 0.9908623099327087, + "sampling/importance_sampling_ratio/min": 0.1831483691930771, + "sampling/sampling_logp_difference/max": 1.6974586248397827, + "sampling/sampling_logp_difference/mean": 0.01822926476597786, + "step": 120, + "step_time": 209.00005609192885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16215.0, + "completions/max_terminated_length": 16215.0, + "completions/mean_length": 9589.25, + "completions/mean_terminated_length": 9589.25, + "completions/min_length": 5802.0, + "completions/min_terminated_length": 5802.0, + "entropy": 0.3083428144454956, + "epoch": 0.1452581032412965, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4060857594013214, + "learning_rate": 5.2e-07, + "loss": -0.0524, + "num_tokens": 42737701.0, + "reward": 0.49280887842178345, + "reward_std": 0.6724991798400879, + "rewards/reward_func/mean": 0.49280887842178345, + "rewards/reward_func/std": 0.6724991798400879, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.6253695487976074, + "sampling/importance_sampling_ratio/mean": 0.9890679717063904, + "sampling/importance_sampling_ratio/min": 0.00019750210049096495, + "sampling/sampling_logp_difference/max": 8.52976131439209, + "sampling/sampling_logp_difference/mean": 0.021406318992376328, + "step": 121, + "step_time": 241.5949507113546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15308.0, + "completions/max_terminated_length": 15308.0, + "completions/mean_length": 10594.34375, + "completions/mean_terminated_length": 10594.34375, + "completions/min_length": 7324.0, + "completions/min_terminated_length": 7324.0, + "entropy": 0.28266207315027714, + "epoch": 0.14645858343337334, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.2750304937362671, + "learning_rate": 5.16e-07, + "loss": 0.0195, + "num_tokens": 43113840.0, + "reward": 0.5235170722007751, + "reward_std": 0.633671224117279, + "rewards/reward_func/mean": 0.5235170722007751, + "rewards/reward_func/std": 0.633671224117279, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4920129776000977, + "sampling/importance_sampling_ratio/mean": 0.9901369214057922, + "sampling/importance_sampling_ratio/min": 0.10062485188245773, + "sampling/sampling_logp_difference/max": 2.296355962753296, + "sampling/sampling_logp_difference/mean": 0.01979176700115204, + "step": 122, + "step_time": 225.33721455116756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15019.0, + "completions/max_terminated_length": 15019.0, + "completions/mean_length": 11032.53125, + "completions/mean_terminated_length": 11032.53125, + "completions/min_length": 5304.0, + "completions/min_terminated_length": 5304.0, + "entropy": 0.26705341041088104, + "epoch": 0.14765906362545017, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3584786653518677, + "learning_rate": 5.12e-07, + "loss": 0.0368, + "num_tokens": 43507401.0, + "reward": 1.9813694953918457, + "reward_std": 2.857267141342163, + "rewards/reward_func/mean": 1.9813694953918457, + "rewards/reward_func/std": 2.857267379760742, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2451746463775635, + "sampling/importance_sampling_ratio/mean": 0.9905773997306824, + "sampling/importance_sampling_ratio/min": 0.07104846835136414, + "sampling/sampling_logp_difference/max": 2.644392967224121, + "sampling/sampling_logp_difference/mean": 0.019043972715735435, + "step": 123, + "step_time": 460.36738448997494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20032.0, + "completions/max_terminated_length": 20032.0, + "completions/mean_length": 11611.53125, + "completions/mean_terminated_length": 11611.53125, + "completions/min_length": 6363.0, + "completions/min_terminated_length": 6363.0, + "entropy": 0.2338911509141326, + "epoch": 0.148859543817527, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3491296172142029, + "learning_rate": 5.079999999999999e-07, + "loss": 0.0364, + "num_tokens": 43926578.0, + "reward": 1.0934959650039673, + "reward_std": 1.0090538263320923, + "rewards/reward_func/mean": 1.1581779718399048, + "rewards/reward_func/std": 0.9206379055976868, + "rewards/soft_overlong_punishment_reward/mean": -0.0646820068359375, + "rewards/soft_overlong_punishment_reward/std": 0.220370814204216, + "sampling/importance_sampling_ratio/max": 1.99778151512146, + "sampling/importance_sampling_ratio/mean": 0.9919837713241577, + "sampling/importance_sampling_ratio/min": 0.01377395261079073, + "sampling/sampling_logp_difference/max": 4.284976005554199, + "sampling/sampling_logp_difference/mean": 0.016360996291041374, + "step": 124, + "step_time": 275.2218638029881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16157.0, + "completions/max_terminated_length": 16157.0, + "completions/mean_length": 10606.0625, + "completions/mean_terminated_length": 10606.0625, + "completions/min_length": 4477.0, + "completions/min_terminated_length": 4477.0, + "entropy": 0.2876010574400425, + "epoch": 0.15006002400960383, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.4879244565963745, + "learning_rate": 5.04e-07, + "loss": 0.0299, + "num_tokens": 44300284.0, + "reward": 8.492034912109375, + "reward_std": 24.353479385375977, + "rewards/reward_func/mean": 8.492034912109375, + "rewards/reward_func/std": 24.35348129272461, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.82053542137146, + "sampling/importance_sampling_ratio/mean": 0.9900239109992981, + "sampling/importance_sampling_ratio/min": 0.2887732684612274, + "sampling/sampling_logp_difference/max": 1.242113471031189, + "sampling/sampling_logp_difference/mean": 0.020131349563598633, + "step": 125, + "step_time": 262.3409889435861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15917.0, + "completions/max_terminated_length": 15917.0, + "completions/mean_length": 9516.4375, + "completions/mean_terminated_length": 9516.4375, + "completions/min_length": 4847.0, + "completions/min_terminated_length": 4847.0, + "entropy": 0.2908124439418316, + "epoch": 0.15126050420168066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38954854011535645, + "learning_rate": 5e-07, + "loss": 0.0245, + "num_tokens": 44634890.0, + "reward": 1.9146395921707153, + "reward_std": 2.8896992206573486, + "rewards/reward_func/mean": 1.9146395921707153, + "rewards/reward_func/std": 2.8896987438201904, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4463016986846924, + "sampling/importance_sampling_ratio/mean": 0.9897171854972839, + "sampling/importance_sampling_ratio/min": 0.3122645914554596, + "sampling/sampling_logp_difference/max": 1.1639044284820557, + "sampling/sampling_logp_difference/mean": 0.020406031981110573, + "step": 126, + "step_time": 262.93810594966635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15624.0, + "completions/max_terminated_length": 15624.0, + "completions/mean_length": 9908.875, + "completions/mean_terminated_length": 9908.875, + "completions/min_length": 6398.0, + "completions/min_terminated_length": 6398.0, + "entropy": 0.3054868131875992, + "epoch": 0.1524609843937575, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3650784492492676, + "learning_rate": 4.96e-07, + "loss": 0.0082, + "num_tokens": 44975030.0, + "reward": 1.5860934257507324, + "reward_std": 2.869269847869873, + "rewards/reward_func/mean": 1.5860934257507324, + "rewards/reward_func/std": 2.869270086288452, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.127547025680542, + "sampling/importance_sampling_ratio/mean": 0.9892829060554504, + "sampling/importance_sampling_ratio/min": 0.21681442856788635, + "sampling/sampling_logp_difference/max": 1.5287134647369385, + "sampling/sampling_logp_difference/mean": 0.02112971805036068, + "step": 127, + "step_time": 216.3248087487882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14126.0, + "completions/max_terminated_length": 14126.0, + "completions/mean_length": 7167.53125, + "completions/mean_terminated_length": 7167.53125, + "completions/min_length": 5186.0, + "completions/min_terminated_length": 5186.0, + "entropy": 0.2729322211816907, + "epoch": 0.15366146458583432, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44890305399894714, + "learning_rate": 4.92e-07, + "loss": 0.0657, + "num_tokens": 45233063.0, + "reward": 3.381237745285034, + "reward_std": 3.5361828804016113, + "rewards/reward_func/mean": 3.381237745285034, + "rewards/reward_func/std": 3.5361826419830322, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.002379894256592, + "sampling/importance_sampling_ratio/mean": 0.9904590845108032, + "sampling/importance_sampling_ratio/min": 0.3677864670753479, + "sampling/sampling_logp_difference/max": 1.0002528429031372, + "sampling/sampling_logp_difference/mean": 0.019061625003814697, + "step": 128, + "step_time": 196.9817350585945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15338.0, + "completions/max_terminated_length": 15338.0, + "completions/mean_length": 7408.9375, + "completions/mean_terminated_length": 7408.9375, + "completions/min_length": 2486.0, + "completions/min_terminated_length": 2486.0, + "entropy": 0.28559087216854095, + "epoch": 0.15486194477791115, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4474518895149231, + "learning_rate": 4.879999999999999e-07, + "loss": 0.0143, + "num_tokens": 45501781.0, + "reward": 1.7653834819793701, + "reward_std": 1.865670919418335, + "rewards/reward_func/mean": 1.7653834819793701, + "rewards/reward_func/std": 1.865670919418335, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.251772165298462, + "sampling/importance_sampling_ratio/mean": 0.9898481369018555, + "sampling/importance_sampling_ratio/min": 0.33571237325668335, + "sampling/sampling_logp_difference/max": 1.0915005207061768, + "sampling/sampling_logp_difference/mean": 0.020369449630379677, + "step": 129, + "step_time": 213.23672278015874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 13311.0, + "completions/mean_length": 10419.03125, + "completions/mean_terminated_length": 9378.2412109375, + "completions/min_length": 6891.0, + "completions/min_terminated_length": 6891.0, + "entropy": 0.288710068911314, + "epoch": 0.15606242496998798, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.37506240606307983, + "learning_rate": 4.839999999999999e-07, + "loss": -0.1003, + "num_tokens": 45864014.0, + "reward": 1.1006951332092285, + "reward_std": 1.99079167842865, + "rewards/reward_func/mean": 1.3180084228515625, + "rewards/reward_func/std": 1.9662554264068604, + "rewards/soft_overlong_punishment_reward/mean": -0.09375, + "rewards/soft_overlong_punishment_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 2.094200849533081, + "sampling/importance_sampling_ratio/mean": 0.9896755218505859, + "sampling/importance_sampling_ratio/min": 3.3655967854429036e-05, + "sampling/sampling_logp_difference/max": 10.299320220947266, + "sampling/sampling_logp_difference/mean": 0.02032674476504326, + "step": 130, + "step_time": 249.37841368792579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13651.0, + "completions/max_terminated_length": 13651.0, + "completions/mean_length": 10875.9375, + "completions/mean_terminated_length": 10875.9375, + "completions/min_length": 6579.0, + "completions/min_terminated_length": 6579.0, + "entropy": 0.27954767271876335, + "epoch": 0.15726290516206481, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4141541123390198, + "learning_rate": 4.8e-07, + "loss": -0.0105, + "num_tokens": 46248332.0, + "reward": 0.5992684364318848, + "reward_std": 0.9581432342529297, + "rewards/reward_func/mean": 0.5992684364318848, + "rewards/reward_func/std": 0.9581431746482849, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2382140159606934, + "sampling/importance_sampling_ratio/mean": 0.990227997303009, + "sampling/importance_sampling_ratio/min": 0.3830912113189697, + "sampling/sampling_logp_difference/max": 0.9594821929931641, + "sampling/sampling_logp_difference/mean": 0.019494500011205673, + "step": 131, + "step_time": 450.12081778328866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11920.0, + "completions/max_terminated_length": 11920.0, + "completions/mean_length": 7238.625, + "completions/mean_terminated_length": 7238.625, + "completions/min_length": 3557.0, + "completions/min_terminated_length": 3557.0, + "entropy": 0.2874604295939207, + "epoch": 0.15846338535414164, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5339971780776978, + "learning_rate": 4.76e-07, + "loss": 0.0164, + "num_tokens": 46507880.0, + "reward": 1.9809516668319702, + "reward_std": 1.8525370359420776, + "rewards/reward_func/mean": 1.9809516668319702, + "rewards/reward_func/std": 1.8525370359420776, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.424567699432373, + "sampling/importance_sampling_ratio/mean": 0.9898762702941895, + "sampling/importance_sampling_ratio/min": 0.3163217008113861, + "sampling/sampling_logp_difference/max": 1.1509954929351807, + "sampling/sampling_logp_difference/mean": 0.020033420994877815, + "step": 132, + "step_time": 163.30515668692533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10824.0, + "completions/max_terminated_length": 10824.0, + "completions/mean_length": 8501.71875, + "completions/mean_terminated_length": 8501.71875, + "completions/min_length": 5532.0, + "completions/min_terminated_length": 5532.0, + "entropy": 0.2722770730033517, + "epoch": 0.15966386554621848, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4209204614162445, + "learning_rate": 4.7199999999999994e-07, + "loss": 0.0654, + "num_tokens": 46813647.0, + "reward": 5.072177886962891, + "reward_std": 4.542591094970703, + "rewards/reward_func/mean": 5.072177886962891, + "rewards/reward_func/std": 4.542591094970703, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.7433615922927856, + "sampling/importance_sampling_ratio/mean": 0.9904061555862427, + "sampling/importance_sampling_ratio/min": 0.09293912351131439, + "sampling/sampling_logp_difference/max": 2.3758106231689453, + "sampling/sampling_logp_difference/mean": 0.01888871379196644, + "step": 133, + "step_time": 183.62756524700671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16072.0, + "completions/max_terminated_length": 16072.0, + "completions/mean_length": 11215.71875, + "completions/mean_terminated_length": 11215.71875, + "completions/min_length": 7240.0, + "completions/min_terminated_length": 7240.0, + "entropy": 0.22843076288700104, + "epoch": 0.1608643457382953, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3479200005531311, + "learning_rate": 4.68e-07, + "loss": -0.0415, + "num_tokens": 47226070.0, + "reward": 0.25698286294937134, + "reward_std": 0.8653408288955688, + "rewards/reward_func/mean": 0.25698286294937134, + "rewards/reward_func/std": 0.8653408288955688, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9921608567237854, + "sampling/importance_sampling_ratio/min": 8.29739210672642e-09, + "sampling/sampling_logp_difference/max": 18.607324600219727, + "sampling/sampling_logp_difference/mean": 0.0165218785405159, + "step": 134, + "step_time": 278.68348298675846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15346.0, + "completions/max_terminated_length": 15346.0, + "completions/mean_length": 9651.1875, + "completions/mean_terminated_length": 9651.1875, + "completions/min_length": 4686.0, + "completions/min_terminated_length": 4686.0, + "entropy": 0.2711868965998292, + "epoch": 0.16206482593037214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3969678580760956, + "learning_rate": 4.64e-07, + "loss": -0.0081, + "num_tokens": 47570140.0, + "reward": 2.622737407684326, + "reward_std": 3.003697633743286, + "rewards/reward_func/mean": 2.622737407684326, + "rewards/reward_func/std": 3.003697395324707, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.7643277645111084, + "sampling/importance_sampling_ratio/mean": 0.990485668182373, + "sampling/importance_sampling_ratio/min": 0.2230183184146881, + "sampling/sampling_logp_difference/max": 1.5005013942718506, + "sampling/sampling_logp_difference/mean": 0.018908120691776276, + "step": 135, + "step_time": 213.85006742121186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11030.0, + "completions/max_terminated_length": 11030.0, + "completions/mean_length": 6975.90625, + "completions/mean_terminated_length": 6975.90625, + "completions/min_length": 3032.0, + "completions/min_terminated_length": 3032.0, + "entropy": 0.307786175981164, + "epoch": 0.16326530612244897, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.47706398367881775, + "learning_rate": 4.6e-07, + "loss": 0.0439, + "num_tokens": 47817641.0, + "reward": 2.0658512115478516, + "reward_std": 1.9194847345352173, + "rewards/reward_func/mean": 2.0658512115478516, + "rewards/reward_func/std": 1.9194847345352173, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.136054754257202, + "sampling/importance_sampling_ratio/mean": 0.9892805218696594, + "sampling/importance_sampling_ratio/min": 1.11404536667876e-15, + "sampling/sampling_logp_difference/max": 34.43077850341797, + "sampling/sampling_logp_difference/mean": 0.021540788933634758, + "step": 136, + "step_time": 174.63315520831384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15303.0, + "completions/max_terminated_length": 15303.0, + "completions/mean_length": 10812.3125, + "completions/mean_terminated_length": 10812.3125, + "completions/min_length": 7491.0, + "completions/min_terminated_length": 7491.0, + "entropy": 0.2458400446921587, + "epoch": 0.1644657863145258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.35999420285224915, + "learning_rate": 4.56e-07, + "loss": 0.0237, + "num_tokens": 48214651.0, + "reward": 2.3839621543884277, + "reward_std": 3.7139806747436523, + "rewards/reward_func/mean": 2.3839621543884277, + "rewards/reward_func/std": 3.7139809131622314, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1656391620635986, + "sampling/importance_sampling_ratio/mean": 0.9914739727973938, + "sampling/importance_sampling_ratio/min": 0.28416091203689575, + "sampling/sampling_logp_difference/max": 1.2582145929336548, + "sampling/sampling_logp_difference/mean": 0.017416341230273247, + "step": 137, + "step_time": 483.0681917546317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15823.0, + "completions/max_terminated_length": 15823.0, + "completions/mean_length": 9706.625, + "completions/mean_terminated_length": 9706.625, + "completions/min_length": 3619.0, + "completions/min_terminated_length": 3619.0, + "entropy": 0.2981583010405302, + "epoch": 0.16566626650660263, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3441935181617737, + "learning_rate": 4.5199999999999997e-07, + "loss": 0.0477, + "num_tokens": 48558839.0, + "reward": 1027.4560546875, + "reward_std": 3014.18603515625, + "rewards/reward_func/mean": 1027.4560546875, + "rewards/reward_func/std": 3014.18603515625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989443838596344, + "sampling/importance_sampling_ratio/min": 0.0003842052537947893, + "sampling/sampling_logp_difference/max": 7.864333629608154, + "sampling/sampling_logp_difference/mean": 0.020724810659885406, + "step": 138, + "step_time": 429.2948167860741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11754.0, + "completions/max_terminated_length": 11754.0, + "completions/mean_length": 9398.84375, + "completions/mean_terminated_length": 9398.84375, + "completions/min_length": 6889.0, + "completions/min_terminated_length": 6889.0, + "entropy": 0.24972048588097095, + "epoch": 0.16686674669867949, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5226235389709473, + "learning_rate": 4.48e-07, + "loss": 0.0647, + "num_tokens": 48899354.0, + "reward": 1.3123189210891724, + "reward_std": 1.0608717203140259, + "rewards/reward_func/mean": 1.3123189210891724, + "rewards/reward_func/std": 1.0608716011047363, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4963085651397705, + "sampling/importance_sampling_ratio/mean": 0.9913288354873657, + "sampling/importance_sampling_ratio/min": 0.014635493978857994, + "sampling/sampling_logp_difference/max": 4.224305629730225, + "sampling/sampling_logp_difference/mean": 0.017568625509738922, + "step": 139, + "step_time": 214.9732713351259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14780.0, + "completions/max_terminated_length": 14780.0, + "completions/mean_length": 10126.9375, + "completions/mean_terminated_length": 10126.9375, + "completions/min_length": 5722.0, + "completions/min_terminated_length": 5722.0, + "entropy": 0.30815622955560684, + "epoch": 0.16806722689075632, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.43311142921447754, + "learning_rate": 4.44e-07, + "loss": 0.0193, + "num_tokens": 49251000.0, + "reward": 2.31791353225708, + "reward_std": 2.4861373901367188, + "rewards/reward_func/mean": 2.31791353225708, + "rewards/reward_func/std": 2.4861373901367188, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9891684055328369, + "sampling/importance_sampling_ratio/min": 0.002064452273771167, + "sampling/sampling_logp_difference/max": 6.18289041519165, + "sampling/sampling_logp_difference/mean": 0.02135949209332466, + "step": 140, + "step_time": 213.63927030388732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15453.0, + "completions/max_terminated_length": 15453.0, + "completions/mean_length": 9498.9375, + "completions/mean_terminated_length": 9498.9375, + "completions/min_length": 5658.0, + "completions/min_terminated_length": 5658.0, + "entropy": 0.23229695297777653, + "epoch": 0.16926770708283315, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.30538368225097656, + "learning_rate": 4.3999999999999997e-07, + "loss": -0.0339, + "num_tokens": 49601214.0, + "reward": 0.6548795700073242, + "reward_std": 0.8164750337600708, + "rewards/reward_func/mean": 0.6548795700073242, + "rewards/reward_func/std": 0.8164750337600708, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2676289081573486, + "sampling/importance_sampling_ratio/mean": 0.9919700622558594, + "sampling/importance_sampling_ratio/min": 0.0972149670124054, + "sampling/sampling_logp_difference/max": 2.3308305740356445, + "sampling/sampling_logp_difference/mean": 0.016680430620908737, + "step": 141, + "step_time": 231.1800272183027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14022.0, + "completions/max_terminated_length": 14022.0, + "completions/mean_length": 8557.125, + "completions/mean_terminated_length": 8557.125, + "completions/min_length": 4786.0, + "completions/min_terminated_length": 4786.0, + "entropy": 0.27586540672928095, + "epoch": 0.17046818727490998, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3966992199420929, + "learning_rate": 4.36e-07, + "loss": -0.0055, + "num_tokens": 49905418.0, + "reward": 0.8080017566680908, + "reward_std": 1.4851527214050293, + "rewards/reward_func/mean": 0.8080017566680908, + "rewards/reward_func/std": 1.4851524829864502, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.329014539718628, + "sampling/importance_sampling_ratio/mean": 0.9903725981712341, + "sampling/importance_sampling_ratio/min": 0.3647924065589905, + "sampling/sampling_logp_difference/max": 1.0084269046783447, + "sampling/sampling_logp_difference/mean": 0.01935010962188244, + "step": 142, + "step_time": 201.8291045529768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10552.0, + "completions/max_terminated_length": 10552.0, + "completions/mean_length": 8153.9375, + "completions/mean_terminated_length": 8153.9375, + "completions/min_length": 5045.0, + "completions/min_terminated_length": 5045.0, + "entropy": 0.260047709569335, + "epoch": 0.1716686674669868, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3805393874645233, + "learning_rate": 4.3199999999999995e-07, + "loss": -0.0278, + "num_tokens": 50202872.0, + "reward": 0.9525775909423828, + "reward_std": 1.2389754056930542, + "rewards/reward_func/mean": 0.9525775909423828, + "rewards/reward_func/std": 1.2389752864837646, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.504868507385254, + "sampling/importance_sampling_ratio/mean": 0.9909122586250305, + "sampling/importance_sampling_ratio/min": 0.055922865867614746, + "sampling/sampling_logp_difference/max": 2.883781909942627, + "sampling/sampling_logp_difference/mean": 0.018256302922964096, + "step": 143, + "step_time": 171.05576233169995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19289.0, + "completions/max_terminated_length": 19289.0, + "completions/mean_length": 10663.5625, + "completions/mean_terminated_length": 10663.5625, + "completions/min_length": 5378.0, + "completions/min_terminated_length": 5378.0, + "entropy": 0.2466154731810093, + "epoch": 0.17286914765906364, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34512680768966675, + "learning_rate": 4.2799999999999997e-07, + "loss": 0.0035, + "num_tokens": 50586930.0, + "reward": 2.0170817375183105, + "reward_std": 5.622786998748779, + "rewards/reward_func/mean": 2.039245128631592, + "rewards/reward_func/std": 5.613083362579346, + "rewards/soft_overlong_punishment_reward/mean": -0.02216339111328125, + "rewards/soft_overlong_punishment_reward/std": 0.12537507712841034, + "sampling/importance_sampling_ratio/max": 2.3612794876098633, + "sampling/importance_sampling_ratio/mean": 0.9915375113487244, + "sampling/importance_sampling_ratio/min": 0.20737797021865845, + "sampling/sampling_logp_difference/max": 1.5732121467590332, + "sampling/sampling_logp_difference/mean": 0.017183100804686546, + "step": 144, + "step_time": 274.66985757567454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14072.0, + "completions/max_terminated_length": 14072.0, + "completions/mean_length": 10114.15625, + "completions/mean_terminated_length": 10114.15625, + "completions/min_length": 5891.0, + "completions/min_terminated_length": 5891.0, + "entropy": 0.2593291876837611, + "epoch": 0.17406962785114047, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4165084958076477, + "learning_rate": 4.24e-07, + "loss": -0.0155, + "num_tokens": 50952575.0, + "reward": 0.6612014174461365, + "reward_std": 0.6770195364952087, + "rewards/reward_func/mean": 0.6612014174461365, + "rewards/reward_func/std": 0.6770195364952087, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3497302532196045, + "sampling/importance_sampling_ratio/mean": 0.9909900426864624, + "sampling/importance_sampling_ratio/min": 0.00037760011036880314, + "sampling/sampling_logp_difference/max": 7.881674766540527, + "sampling/sampling_logp_difference/mean": 0.018293218687176704, + "step": 145, + "step_time": 220.56290533288848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15097.0, + "completions/max_terminated_length": 15097.0, + "completions/mean_length": 10543.625, + "completions/mean_terminated_length": 10543.625, + "completions/min_length": 5699.0, + "completions/min_terminated_length": 5699.0, + "entropy": 0.2614777237176895, + "epoch": 0.1752701080432173, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3072114884853363, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.0314, + "num_tokens": 51330571.0, + "reward": 0.5684531927108765, + "reward_std": 0.9393928647041321, + "rewards/reward_func/mean": 0.5684531927108765, + "rewards/reward_func/std": 0.9393928050994873, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.8473550081253052, + "sampling/importance_sampling_ratio/mean": 0.9907210469245911, + "sampling/importance_sampling_ratio/min": 0.07700582593679428, + "sampling/sampling_logp_difference/max": 2.5638742446899414, + "sampling/sampling_logp_difference/mean": 0.01846243068575859, + "step": 146, + "step_time": 453.85375829122495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13158.0, + "completions/max_terminated_length": 13158.0, + "completions/mean_length": 8469.71875, + "completions/mean_terminated_length": 8469.71875, + "completions/min_length": 3331.0, + "completions/min_terminated_length": 3331.0, + "entropy": 0.29093294590711594, + "epoch": 0.17647058823529413, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4070035517215729, + "learning_rate": 4.1599999999999997e-07, + "loss": 0.0461, + "num_tokens": 51637050.0, + "reward": 2.5153932571411133, + "reward_std": 2.769888401031494, + "rewards/reward_func/mean": 2.5153932571411133, + "rewards/reward_func/std": 2.769888401031494, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0668880939483643, + "sampling/importance_sampling_ratio/mean": 0.9898422956466675, + "sampling/importance_sampling_ratio/min": 0.24710887670516968, + "sampling/sampling_logp_difference/max": 1.3979263305664062, + "sampling/sampling_logp_difference/mean": 0.0202447809278965, + "step": 147, + "step_time": 189.9260320410831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14203.0, + "completions/max_terminated_length": 14203.0, + "completions/mean_length": 8437.125, + "completions/mean_terminated_length": 8437.125, + "completions/min_length": 4104.0, + "completions/min_terminated_length": 4104.0, + "entropy": 0.2681101858615875, + "epoch": 0.17767106842737096, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.39123448729515076, + "learning_rate": 4.12e-07, + "loss": 0.022, + "num_tokens": 51937598.0, + "reward": 2.7620649337768555, + "reward_std": 3.608797073364258, + "rewards/reward_func/mean": 2.7620649337768555, + "rewards/reward_func/std": 3.608797073364258, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.5222227573394775, + "sampling/importance_sampling_ratio/mean": 0.9908010959625244, + "sampling/importance_sampling_ratio/min": 0.23630328476428986, + "sampling/sampling_logp_difference/max": 1.4426392316818237, + "sampling/sampling_logp_difference/mean": 0.018574317917227745, + "step": 148, + "step_time": 205.4168942749966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17113.0, + "completions/max_terminated_length": 17113.0, + "completions/mean_length": 9732.375, + "completions/mean_terminated_length": 9732.375, + "completions/min_length": 5745.0, + "completions/min_terminated_length": 5745.0, + "entropy": 0.28236152697354555, + "epoch": 0.1788715486194478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41433754563331604, + "learning_rate": 4.0799999999999995e-07, + "loss": -0.0364, + "num_tokens": 52280586.0, + "reward": 1.7899069786071777, + "reward_std": 2.3258073329925537, + "rewards/reward_func/mean": 1.799375057220459, + "rewards/reward_func/std": 2.324876070022583, + "rewards/soft_overlong_punishment_reward/mean": -0.00946807861328125, + "rewards/soft_overlong_punishment_reward/std": 0.03785913065075874, + "sampling/importance_sampling_ratio/max": 1.7942250967025757, + "sampling/importance_sampling_ratio/mean": 0.9903931021690369, + "sampling/importance_sampling_ratio/min": 0.028741903603076935, + "sampling/sampling_logp_difference/max": 3.5493991374969482, + "sampling/sampling_logp_difference/mean": 0.01913137547671795, + "step": 149, + "step_time": 246.32968526915647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11989.0, + "completions/max_terminated_length": 11989.0, + "completions/mean_length": 6893.03125, + "completions/mean_terminated_length": 6893.03125, + "completions/min_length": 2952.0, + "completions/min_terminated_length": 2952.0, + "entropy": 0.3159897867590189, + "epoch": 0.18007202881152462, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4671184718608856, + "learning_rate": 4.04e-07, + "loss": 0.022, + "num_tokens": 52520571.0, + "reward": 2.152081251144409, + "reward_std": 2.514601707458496, + "rewards/reward_func/mean": 2.152081251144409, + "rewards/reward_func/std": 2.514601707458496, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3258213996887207, + "sampling/importance_sampling_ratio/mean": 0.9888448715209961, + "sampling/importance_sampling_ratio/min": 0.2504069209098816, + "sampling/sampling_logp_difference/max": 1.384667992591858, + "sampling/sampling_logp_difference/mean": 0.021897951140999794, + "step": 150, + "step_time": 195.26920710073318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12887.0, + "completions/max_terminated_length": 12887.0, + "completions/mean_length": 8146.09375, + "completions/mean_terminated_length": 8146.09375, + "completions/min_length": 4733.0, + "completions/min_terminated_length": 4733.0, + "entropy": 0.2663646712899208, + "epoch": 0.18127250900360145, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4627015292644501, + "learning_rate": 4e-07, + "loss": -0.0405, + "num_tokens": 52819142.0, + "reward": 1.819735050201416, + "reward_std": 2.4615325927734375, + "rewards/reward_func/mean": 1.819735050201416, + "rewards/reward_func/std": 2.4615325927734375, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3554580211639404, + "sampling/importance_sampling_ratio/mean": 0.9907447695732117, + "sampling/importance_sampling_ratio/min": 0.12350412458181381, + "sampling/sampling_logp_difference/max": 2.0914807319641113, + "sampling/sampling_logp_difference/mean": 0.01854807324707508, + "step": 151, + "step_time": 201.55244890286122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16359.0, + "completions/max_terminated_length": 16359.0, + "completions/mean_length": 9608.84375, + "completions/mean_terminated_length": 9608.84375, + "completions/min_length": 4375.0, + "completions/min_terminated_length": 4375.0, + "entropy": 0.2643621703609824, + "epoch": 0.18247298919567828, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4149344563484192, + "learning_rate": 3.96e-07, + "loss": 0.0057, + "num_tokens": 53164609.0, + "reward": 0.8813707828521729, + "reward_std": 1.7989546060562134, + "rewards/reward_func/mean": 0.8813707828521729, + "rewards/reward_func/std": 1.7989544868469238, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.488729476928711, + "sampling/importance_sampling_ratio/mean": 0.9908096790313721, + "sampling/importance_sampling_ratio/min": 0.10281366854906082, + "sampling/sampling_logp_difference/max": 2.274837017059326, + "sampling/sampling_logp_difference/mean": 0.018455805256962776, + "step": 152, + "step_time": 226.86713050387334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 10407.0, + "completions/mean_length": 8275.34375, + "completions/mean_terminated_length": 7881.64501953125, + "completions/min_length": 5786.0, + "completions/min_terminated_length": 5786.0, + "entropy": 0.2822192329913378, + "epoch": 0.1836734693877551, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38609081506729126, + "learning_rate": 3.92e-07, + "loss": -0.0484, + "num_tokens": 53458380.0, + "reward": 2.3562521934509277, + "reward_std": 3.1254520416259766, + "rewards/reward_func/mean": 2.4645183086395264, + "rewards/reward_func/std": 3.11552095413208, + "rewards/soft_overlong_punishment_reward/mean": -0.03125, + "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.7898873090744019, + "sampling/importance_sampling_ratio/mean": 0.990231454372406, + "sampling/importance_sampling_ratio/min": 0.25040626525878906, + "sampling/sampling_logp_difference/max": 1.384670615196228, + "sampling/sampling_logp_difference/mean": 0.019599847495555878, + "step": 153, + "step_time": 236.71146991685964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14804.0, + "completions/max_terminated_length": 14804.0, + "completions/mean_length": 9698.5625, + "completions/mean_terminated_length": 9698.5625, + "completions/min_length": 4843.0, + "completions/min_terminated_length": 4843.0, + "entropy": 0.2661178121343255, + "epoch": 0.18487394957983194, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38603511452674866, + "learning_rate": 3.88e-07, + "loss": -0.0049, + "num_tokens": 53810886.0, + "reward": 0.7766771912574768, + "reward_std": 1.2446486949920654, + "rewards/reward_func/mean": 0.7766771912574768, + "rewards/reward_func/std": 1.2446486949920654, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.604177951812744, + "sampling/importance_sampling_ratio/mean": 0.9907273054122925, + "sampling/importance_sampling_ratio/min": 2.992864028783515e-05, + "sampling/sampling_logp_difference/max": 10.416694641113281, + "sampling/sampling_logp_difference/mean": 0.018732445314526558, + "step": 154, + "step_time": 311.80545009789057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13282.0, + "completions/max_terminated_length": 13282.0, + "completions/mean_length": 10236.65625, + "completions/mean_terminated_length": 10236.65625, + "completions/min_length": 6620.0, + "completions/min_terminated_length": 6620.0, + "entropy": 0.23385445773601532, + "epoch": 0.18607442977190877, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34655675292015076, + "learning_rate": 3.84e-07, + "loss": 0.0323, + "num_tokens": 54183971.0, + "reward": 1.4783434867858887, + "reward_std": 4.052839756011963, + "rewards/reward_func/mean": 1.4783434867858887, + "rewards/reward_func/std": 4.052839756011963, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1445651054382324, + "sampling/importance_sampling_ratio/mean": 0.9918794631958008, + "sampling/importance_sampling_ratio/min": 0.05829664319753647, + "sampling/sampling_logp_difference/max": 2.8422107696533203, + "sampling/sampling_logp_difference/mean": 0.0164381992071867, + "step": 155, + "step_time": 215.27548193011899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13371.0, + "completions/max_terminated_length": 13371.0, + "completions/mean_length": 8997.625, + "completions/mean_terminated_length": 8997.625, + "completions/min_length": 5840.0, + "completions/min_terminated_length": 5840.0, + "entropy": 0.23230466898530722, + "epoch": 0.1872749099639856, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3713797628879547, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.0149, + "num_tokens": 54512903.0, + "reward": 1.508521318435669, + "reward_std": 2.5881083011627197, + "rewards/reward_func/mean": 1.508521318435669, + "rewards/reward_func/std": 2.5881083011627197, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9918398857116699, + "sampling/importance_sampling_ratio/min": 0.12671518325805664, + "sampling/sampling_logp_difference/max": 2.0658133029937744, + "sampling/sampling_logp_difference/mean": 0.016411125659942627, + "step": 156, + "step_time": 255.3710762731498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12916.0, + "completions/max_terminated_length": 12916.0, + "completions/mean_length": 8303.0625, + "completions/mean_terminated_length": 8303.0625, + "completions/min_length": 3105.0, + "completions/min_terminated_length": 3105.0, + "entropy": 0.2881615702062845, + "epoch": 0.18847539015606243, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39465340971946716, + "learning_rate": 3.76e-07, + "loss": 0.0027, + "num_tokens": 54806233.0, + "reward": 2.848686456680298, + "reward_std": 3.52091646194458, + "rewards/reward_func/mean": 2.848686456680298, + "rewards/reward_func/std": 3.520916700363159, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2623817920684814, + "sampling/importance_sampling_ratio/mean": 0.9897094964981079, + "sampling/importance_sampling_ratio/min": 0.13919180631637573, + "sampling/sampling_logp_difference/max": 1.9719023704528809, + "sampling/sampling_logp_difference/mean": 0.020325226709246635, + "step": 157, + "step_time": 223.82761657808442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13680.0, + "completions/max_terminated_length": 13680.0, + "completions/mean_length": 8154.1875, + "completions/mean_terminated_length": 8154.1875, + "completions/min_length": 4297.0, + "completions/min_terminated_length": 4297.0, + "entropy": 0.31936432234942913, + "epoch": 0.18967587034813926, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.40818971395492554, + "learning_rate": 3.72e-07, + "loss": -0.0041, + "num_tokens": 55087247.0, + "reward": 0.5234838724136353, + "reward_std": 0.6490806937217712, + "rewards/reward_func/mean": 0.5234838724136353, + "rewards/reward_func/std": 0.6490806341171265, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9888770580291748, + "sampling/importance_sampling_ratio/min": 0.18931546807289124, + "sampling/sampling_logp_difference/max": 1.6643404960632324, + "sampling/sampling_logp_difference/mean": 0.021932730451226234, + "step": 158, + "step_time": 218.13758707989473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15527.0, + "completions/max_terminated_length": 15527.0, + "completions/mean_length": 10728.09375, + "completions/mean_terminated_length": 10728.09375, + "completions/min_length": 4513.0, + "completions/min_terminated_length": 4513.0, + "entropy": 0.3004476949572563, + "epoch": 0.1908763505402161, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3766877353191376, + "learning_rate": 3.6799999999999996e-07, + "loss": 0.0295, + "num_tokens": 55460634.0, + "reward": 18.52477264404297, + "reward_std": 40.702205657958984, + "rewards/reward_func/mean": 18.52477264404297, + "rewards/reward_func/std": 40.702205657958984, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.10101580619812, + "sampling/importance_sampling_ratio/mean": 0.9893911480903625, + "sampling/importance_sampling_ratio/min": 0.34575095772743225, + "sampling/sampling_logp_difference/max": 1.0620365142822266, + "sampling/sampling_logp_difference/mean": 0.021128207445144653, + "step": 159, + "step_time": 454.3971255911747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16856.0, + "completions/max_terminated_length": 16856.0, + "completions/mean_length": 10528.46875, + "completions/mean_terminated_length": 10528.46875, + "completions/min_length": 7584.0, + "completions/min_terminated_length": 7584.0, + "entropy": 0.24533494841307402, + "epoch": 0.19207683073229292, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3707125782966614, + "learning_rate": 3.64e-07, + "loss": 0.04, + "num_tokens": 55848537.0, + "reward": 1.3989126682281494, + "reward_std": 0.9870246648788452, + "rewards/reward_func/mean": 1.4025137424468994, + "rewards/reward_func/std": 0.9815170168876648, + "rewards/soft_overlong_punishment_reward/mean": -0.00360107421875, + "rewards/soft_overlong_punishment_reward/std": 0.02037075161933899, + "sampling/importance_sampling_ratio/max": 2.11053729057312, + "sampling/importance_sampling_ratio/mean": 0.9915527105331421, + "sampling/importance_sampling_ratio/min": 0.2704220414161682, + "sampling/sampling_logp_difference/max": 1.3077714443206787, + "sampling/sampling_logp_difference/mean": 0.017412563785910606, + "step": 160, + "step_time": 272.99579531908967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10456.0, + "completions/max_terminated_length": 10456.0, + "completions/mean_length": 6813.90625, + "completions/mean_terminated_length": 6813.90625, + "completions/min_length": 4551.0, + "completions/min_terminated_length": 4551.0, + "entropy": 0.27734176721423864, + "epoch": 0.19327731092436976, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.48373842239379883, + "learning_rate": 3.6e-07, + "loss": -0.0433, + "num_tokens": 56091246.0, + "reward": 0.9251768589019775, + "reward_std": 1.2734801769256592, + "rewards/reward_func/mean": 0.9251768589019775, + "rewards/reward_func/std": 1.2734800577163696, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.068244218826294, + "sampling/importance_sampling_ratio/mean": 0.990391194820404, + "sampling/importance_sampling_ratio/min": 0.18318888545036316, + "sampling/sampling_logp_difference/max": 1.697237491607666, + "sampling/sampling_logp_difference/mean": 0.01954927295446396, + "step": 161, + "step_time": 222.0121338494355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15822.0, + "completions/max_terminated_length": 15822.0, + "completions/mean_length": 9338.15625, + "completions/mean_terminated_length": 9338.15625, + "completions/min_length": 4551.0, + "completions/min_terminated_length": 4551.0, + "entropy": 0.25283763464540243, + "epoch": 0.19447779111644659, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3540862202644348, + "learning_rate": 3.5599999999999996e-07, + "loss": -0.0236, + "num_tokens": 56431251.0, + "reward": 2.1091599464416504, + "reward_std": 3.2207794189453125, + "rewards/reward_func/mean": 2.1091599464416504, + "rewards/reward_func/std": 3.2207798957824707, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2128798961639404, + "sampling/importance_sampling_ratio/mean": 0.9914829730987549, + "sampling/importance_sampling_ratio/min": 0.43283000588417053, + "sampling/sampling_logp_difference/max": 0.8374102115631104, + "sampling/sampling_logp_difference/mean": 0.017802974209189415, + "step": 162, + "step_time": 242.35264018306043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13122.0, + "completions/max_terminated_length": 13122.0, + "completions/mean_length": 9301.21875, + "completions/mean_terminated_length": 9301.21875, + "completions/min_length": 6263.0, + "completions/min_terminated_length": 6263.0, + "entropy": 0.2940525095909834, + "epoch": 0.19567827130852342, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39872726798057556, + "learning_rate": 3.52e-07, + "loss": 0.0262, + "num_tokens": 56764610.0, + "reward": 1.4544447660446167, + "reward_std": 1.857795238494873, + "rewards/reward_func/mean": 1.4544447660446167, + "rewards/reward_func/std": 1.857795238494873, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9896678924560547, + "sampling/importance_sampling_ratio/min": 0.36473485827445984, + "sampling/sampling_logp_difference/max": 1.1077795028686523, + "sampling/sampling_logp_difference/mean": 0.020575741305947304, + "step": 163, + "step_time": 434.3237003698014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13825.0, + "completions/max_terminated_length": 13825.0, + "completions/mean_length": 9424.21875, + "completions/mean_terminated_length": 9424.21875, + "completions/min_length": 5781.0, + "completions/min_terminated_length": 5781.0, + "entropy": 0.3086765222251415, + "epoch": 0.19687875150060025, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40642544627189636, + "learning_rate": 3.4799999999999994e-07, + "loss": 0.034, + "num_tokens": 57091961.0, + "reward": 66378.796875, + "reward_std": 375489.625, + "rewards/reward_func/mean": 66378.796875, + "rewards/reward_func/std": 375489.625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.98918217420578, + "sampling/importance_sampling_ratio/min": 0.039938684552907944, + "sampling/sampling_logp_difference/max": 3.220409870147705, + "sampling/sampling_logp_difference/mean": 0.02147614024579525, + "step": 164, + "step_time": 258.5584741720231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16400.0, + "completions/max_terminated_length": 16400.0, + "completions/mean_length": 9196.0, + "completions/mean_terminated_length": 9196.0, + "completions/min_length": 3739.0, + "completions/min_terminated_length": 3739.0, + "entropy": 0.2833999376744032, + "epoch": 0.19807923169267708, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0096226930618286, + "learning_rate": 3.4399999999999996e-07, + "loss": -0.0008, + "num_tokens": 57423017.0, + "reward": 0.45536255836486816, + "reward_std": 0.5211973190307617, + "rewards/reward_func/mean": 0.45548462867736816, + "rewards/reward_func/std": 0.521098792552948, + "rewards/soft_overlong_punishment_reward/mean": -0.0001220703125, + "rewards/soft_overlong_punishment_reward/std": 0.0006905339541845024, + "sampling/importance_sampling_ratio/max": 2.0532639026641846, + "sampling/importance_sampling_ratio/mean": 0.9901537895202637, + "sampling/importance_sampling_ratio/min": 0.2596132457256317, + "sampling/sampling_logp_difference/max": 1.348562240600586, + "sampling/sampling_logp_difference/mean": 0.019828852266073227, + "step": 165, + "step_time": 268.0153879421996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15331.0, + "completions/max_terminated_length": 15331.0, + "completions/mean_length": 10289.25, + "completions/mean_terminated_length": 10289.25, + "completions/min_length": 7699.0, + "completions/min_terminated_length": 7699.0, + "entropy": 0.2959921294823289, + "epoch": 0.1992797118847539, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3890758454799652, + "learning_rate": 3.4000000000000003e-07, + "loss": -0.0129, + "num_tokens": 57786057.0, + "reward": 0.2713179886341095, + "reward_std": 0.6150152087211609, + "rewards/reward_func/mean": 0.2713179886341095, + "rewards/reward_func/std": 0.6150152087211609, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9894802570343018, + "sampling/importance_sampling_ratio/min": 0.03638508915901184, + "sampling/sampling_logp_difference/max": 3.313596248626709, + "sampling/sampling_logp_difference/mean": 0.02085329219698906, + "step": 166, + "step_time": 255.01892217074055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16600.0, + "completions/max_terminated_length": 16600.0, + "completions/mean_length": 9496.1875, + "completions/mean_terminated_length": 9496.1875, + "completions/min_length": 5800.0, + "completions/min_terminated_length": 5800.0, + "entropy": 0.2720292443409562, + "epoch": 0.20048019207683074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3967978358268738, + "learning_rate": 3.36e-07, + "loss": 0.0574, + "num_tokens": 58128367.0, + "reward": 2.866912841796875, + "reward_std": 3.3676812648773193, + "rewards/reward_func/mean": 2.868560791015625, + "rewards/reward_func/std": 3.367079496383667, + "rewards/soft_overlong_punishment_reward/mean": -0.00164794921875, + "rewards/soft_overlong_punishment_reward/std": 0.009322208352386951, + "sampling/importance_sampling_ratio/max": 2.074486494064331, + "sampling/importance_sampling_ratio/mean": 0.9905571937561035, + "sampling/importance_sampling_ratio/min": 0.25333765149116516, + "sampling/sampling_logp_difference/max": 1.3730320930480957, + "sampling/sampling_logp_difference/mean": 0.019127877429127693, + "step": 167, + "step_time": 239.48420156200882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12687.0, + "completions/max_terminated_length": 12687.0, + "completions/mean_length": 8358.1875, + "completions/mean_terminated_length": 8358.1875, + "completions/min_length": 5265.0, + "completions/min_terminated_length": 5265.0, + "entropy": 0.28812102414667606, + "epoch": 0.20168067226890757, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41675513982772827, + "learning_rate": 3.32e-07, + "loss": 0.0333, + "num_tokens": 58430437.0, + "reward": 2.597444534301758, + "reward_std": 3.35589861869812, + "rewards/reward_func/mean": 2.597444534301758, + "rewards/reward_func/std": 3.355898857116699, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.186401128768921, + "sampling/importance_sampling_ratio/mean": 0.9898570775985718, + "sampling/importance_sampling_ratio/min": 0.055154554545879364, + "sampling/sampling_logp_difference/max": 2.897615909576416, + "sampling/sampling_logp_difference/mean": 0.020029492676258087, + "step": 168, + "step_time": 226.04919587192126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12104.0, + "completions/max_terminated_length": 12104.0, + "completions/mean_length": 8018.03125, + "completions/mean_terminated_length": 8018.03125, + "completions/min_length": 3307.0, + "completions/min_terminated_length": 3307.0, + "entropy": 0.3021853733807802, + "epoch": 0.2028811524609844, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4631645083427429, + "learning_rate": 3.28e-07, + "loss": 0.0106, + "num_tokens": 58713550.0, + "reward": 0.6359751224517822, + "reward_std": 0.6328838467597961, + "rewards/reward_func/mean": 0.6359751224517822, + "rewards/reward_func/std": 0.6328837871551514, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1259548664093018, + "sampling/importance_sampling_ratio/mean": 0.9894533157348633, + "sampling/importance_sampling_ratio/min": 0.005165865179151297, + "sampling/sampling_logp_difference/max": 5.265682697296143, + "sampling/sampling_logp_difference/mean": 0.020912254229187965, + "step": 169, + "step_time": 182.4748994258698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12901.0, + "completions/max_terminated_length": 12901.0, + "completions/mean_length": 9577.5625, + "completions/mean_terminated_length": 9577.5625, + "completions/min_length": 5991.0, + "completions/min_terminated_length": 5991.0, + "entropy": 0.2918419409543276, + "epoch": 0.20408163265306123, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4238099157810211, + "learning_rate": 3.24e-07, + "loss": 0.0158, + "num_tokens": 59055152.0, + "reward": 1.1994872093200684, + "reward_std": 1.4579551219940186, + "rewards/reward_func/mean": 1.1994872093200684, + "rewards/reward_func/std": 1.457955241203308, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3555917739868164, + "sampling/importance_sampling_ratio/mean": 0.9899040460586548, + "sampling/importance_sampling_ratio/min": 0.15884944796562195, + "sampling/sampling_logp_difference/max": 1.8397983312606812, + "sampling/sampling_logp_difference/mean": 0.020075272768735886, + "step": 170, + "step_time": 205.09224270377308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11374.0, + "completions/max_terminated_length": 11374.0, + "completions/mean_length": 7372.3125, + "completions/mean_terminated_length": 7372.3125, + "completions/min_length": 2447.0, + "completions/min_terminated_length": 2447.0, + "entropy": 0.2957879761233926, + "epoch": 0.20528211284513806, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4720025956630707, + "learning_rate": 3.2e-07, + "loss": 0.0587, + "num_tokens": 59316858.0, + "reward": 2.2769434452056885, + "reward_std": 2.0495495796203613, + "rewards/reward_func/mean": 2.2769434452056885, + "rewards/reward_func/std": 2.0495495796203613, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989801287651062, + "sampling/importance_sampling_ratio/min": 0.36141398549079895, + "sampling/sampling_logp_difference/max": 1.1415172815322876, + "sampling/sampling_logp_difference/mean": 0.020276790484786034, + "step": 171, + "step_time": 171.338580377982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13035.0, + "completions/max_terminated_length": 13035.0, + "completions/mean_length": 9139.9375, + "completions/mean_terminated_length": 9139.9375, + "completions/min_length": 6085.0, + "completions/min_terminated_length": 6085.0, + "entropy": 0.265663193538785, + "epoch": 0.2064825930372149, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4693741202354431, + "learning_rate": 3.1599999999999997e-07, + "loss": 0.0361, + "num_tokens": 59651160.0, + "reward": 1.905058741569519, + "reward_std": 3.157362699508667, + "rewards/reward_func/mean": 1.905058741569519, + "rewards/reward_func/std": 3.157362699508667, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.7062559127807617, + "sampling/importance_sampling_ratio/mean": 0.9907447695732117, + "sampling/importance_sampling_ratio/min": 0.05720826983451843, + "sampling/sampling_logp_difference/max": 2.8610568046569824, + "sampling/sampling_logp_difference/mean": 0.018653716892004013, + "step": 172, + "step_time": 397.120246386854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14177.0, + "completions/max_terminated_length": 14177.0, + "completions/mean_length": 9426.0625, + "completions/mean_terminated_length": 9426.0625, + "completions/min_length": 5614.0, + "completions/min_terminated_length": 5614.0, + "entropy": 0.2500729188323021, + "epoch": 0.20768307322929172, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.2702878415584564, + "learning_rate": 3.12e-07, + "loss": 0.0171, + "num_tokens": 59992778.0, + "reward": 1.3135590553283691, + "reward_std": 2.5076005458831787, + "rewards/reward_func/mean": 1.3135590553283691, + "rewards/reward_func/std": 2.5076003074645996, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.058269500732422, + "sampling/importance_sampling_ratio/mean": 0.9914534687995911, + "sampling/importance_sampling_ratio/min": 0.10936449468135834, + "sampling/sampling_logp_difference/max": 2.213068962097168, + "sampling/sampling_logp_difference/mean": 0.01742243394255638, + "step": 173, + "step_time": 239.5976502839476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12497.0, + "completions/max_terminated_length": 12497.0, + "completions/mean_length": 8828.0, + "completions/mean_terminated_length": 8828.0, + "completions/min_length": 6421.0, + "completions/min_terminated_length": 6421.0, + "entropy": 0.27684462163597345, + "epoch": 0.20888355342136855, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3942298889160156, + "learning_rate": 3.08e-07, + "loss": 0.0059, + "num_tokens": 60315146.0, + "reward": 0.6179296970367432, + "reward_std": 0.6792400479316711, + "rewards/reward_func/mean": 0.6179296970367432, + "rewards/reward_func/std": 0.6792401075363159, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9904294013977051, + "sampling/importance_sampling_ratio/min": 0.2878604829311371, + "sampling/sampling_logp_difference/max": 1.245279312133789, + "sampling/sampling_logp_difference/mean": 0.01934416964650154, + "step": 174, + "step_time": 214.77978121873457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17360.0, + "completions/max_terminated_length": 17360.0, + "completions/mean_length": 9081.84375, + "completions/mean_terminated_length": 9081.84375, + "completions/min_length": 4942.0, + "completions/min_terminated_length": 4942.0, + "entropy": 0.2940607778728008, + "epoch": 0.21008403361344538, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4100255072116852, + "learning_rate": 3.0399999999999997e-07, + "loss": 0.0503, + "num_tokens": 60635101.0, + "reward": 1.2948029041290283, + "reward_std": 1.4059300422668457, + "rewards/reward_func/mean": 1.3022491931915283, + "rewards/reward_func/std": 1.3984326124191284, + "rewards/soft_overlong_punishment_reward/mean": -0.0074462890625, + "rewards/soft_overlong_punishment_reward/std": 0.04212257266044617, + "sampling/importance_sampling_ratio/max": 2.523259162902832, + "sampling/importance_sampling_ratio/mean": 0.9897422790527344, + "sampling/importance_sampling_ratio/min": 0.2878084182739258, + "sampling/sampling_logp_difference/max": 1.2454602718353271, + "sampling/sampling_logp_difference/mean": 0.020703017711639404, + "step": 175, + "step_time": 467.0385398848448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 20183.0, + "completions/mean_length": 11330.46875, + "completions/mean_terminated_length": 9219.0390625, + "completions/min_length": 6454.0, + "completions/min_terminated_length": 6454.0, + "entropy": 0.2567200744524598, + "epoch": 0.2112845138055222, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.33059075474739075, + "learning_rate": 3e-07, + "loss": -0.0585, + "num_tokens": 61020372.0, + "reward": 1.0467814207077026, + "reward_std": 1.7194699048995972, + "rewards/reward_func/mean": 1.55478835105896, + "rewards/reward_func/std": 1.5100483894348145, + "rewards/soft_overlong_punishment_reward/mean": -0.21648406982421875, + "rewards/soft_overlong_punishment_reward/std": 0.41583752632141113, + "sampling/importance_sampling_ratio/max": 2.49721097946167, + "sampling/importance_sampling_ratio/mean": 0.9905225038528442, + "sampling/importance_sampling_ratio/min": 0.2589978873729706, + "sampling/sampling_logp_difference/max": 1.3509353399276733, + "sampling/sampling_logp_difference/mean": 0.019211364910006523, + "step": 176, + "step_time": 262.3387938350206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13843.0, + "completions/max_terminated_length": 13843.0, + "completions/mean_length": 9019.90625, + "completions/mean_terminated_length": 9019.90625, + "completions/min_length": 5026.0, + "completions/min_terminated_length": 5026.0, + "entropy": 0.26270918548107147, + "epoch": 0.21248499399759904, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40801480412483215, + "learning_rate": 2.9599999999999995e-07, + "loss": -0.0516, + "num_tokens": 61349169.0, + "reward": 0.7610185146331787, + "reward_std": 0.6794523000717163, + "rewards/reward_func/mean": 0.7610185146331787, + "rewards/reward_func/std": 0.6794522404670715, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9909648895263672, + "sampling/importance_sampling_ratio/min": 0.45922839641571045, + "sampling/sampling_logp_difference/max": 1.4038057327270508, + "sampling/sampling_logp_difference/mean": 0.0184706449508667, + "step": 177, + "step_time": 210.60857710335404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13779.0, + "completions/max_terminated_length": 13779.0, + "completions/mean_length": 8056.8125, + "completions/mean_terminated_length": 8056.8125, + "completions/min_length": 4695.0, + "completions/min_terminated_length": 4695.0, + "entropy": 0.2336973724886775, + "epoch": 0.21368547418967587, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4138389527797699, + "learning_rate": 2.9199999999999997e-07, + "loss": 0.0636, + "num_tokens": 61643739.0, + "reward": 0.7230340242385864, + "reward_std": 1.2502714395523071, + "rewards/reward_func/mean": 0.7230340242385864, + "rewards/reward_func/std": 1.2502713203430176, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.6206603050231934, + "sampling/importance_sampling_ratio/mean": 0.9920225143432617, + "sampling/importance_sampling_ratio/min": 0.001480705919675529, + "sampling/sampling_logp_difference/max": 6.5152363777160645, + "sampling/sampling_logp_difference/mean": 0.016540315002202988, + "step": 178, + "step_time": 223.01532334729563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 20367.0, + "completions/max_terminated_length": 20367.0, + "completions/mean_length": 9921.0625, + "completions/mean_terminated_length": 9921.0625, + "completions/min_length": 6193.0, + "completions/min_terminated_length": 6193.0, + "entropy": 0.2881326600909233, + "epoch": 0.2148859543817527, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.34104132652282715, + "learning_rate": 2.88e-07, + "loss": 0.05, + "num_tokens": 61997757.0, + "reward": 1.612834095954895, + "reward_std": 2.867422580718994, + "rewards/reward_func/mean": 1.6432219743728638, + "rewards/reward_func/std": 2.8589067459106445, + "rewards/soft_overlong_punishment_reward/mean": -0.03038787841796875, + "rewards/soft_overlong_punishment_reward/std": 0.17189979553222656, + "sampling/importance_sampling_ratio/max": 2.565842866897583, + "sampling/importance_sampling_ratio/mean": 0.9899250268936157, + "sampling/importance_sampling_ratio/min": 0.11633654683828354, + "sampling/sampling_logp_difference/max": 2.1512680053710938, + "sampling/sampling_logp_difference/mean": 0.020425235852599144, + "step": 179, + "step_time": 249.0168090411462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12388.0, + "completions/max_terminated_length": 12388.0, + "completions/mean_length": 6812.59375, + "completions/mean_terminated_length": 6812.59375, + "completions/min_length": 2401.0, + "completions/min_terminated_length": 2401.0, + "entropy": 0.3213646560907364, + "epoch": 0.21608643457382953, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.510503888130188, + "learning_rate": 2.8399999999999995e-07, + "loss": -0.0106, + "num_tokens": 62243096.0, + "reward": 1.7355564832687378, + "reward_std": 1.7355234622955322, + "rewards/reward_func/mean": 1.7355564832687378, + "rewards/reward_func/std": 1.7355234622955322, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2783377170562744, + "sampling/importance_sampling_ratio/mean": 0.9888811707496643, + "sampling/importance_sampling_ratio/min": 0.27378326654434204, + "sampling/sampling_logp_difference/max": 1.2954185009002686, + "sampling/sampling_logp_difference/mean": 0.022066859528422356, + "step": 180, + "step_time": 233.19768030114938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 17498.0, + "completions/mean_length": 10577.9375, + "completions/mean_terminated_length": 10258.515625, + "completions/min_length": 5877.0, + "completions/min_terminated_length": 5877.0, + "entropy": 0.2779244324192405, + "epoch": 0.21728691476590636, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.350612074136734, + "learning_rate": 2.8e-07, + "loss": -0.0024, + "num_tokens": 62611454.0, + "reward": 2.4395365715026855, + "reward_std": 4.4411797523498535, + "rewards/reward_func/mean": 2.479285717010498, + "rewards/reward_func/std": 4.416739463806152, + "rewards/soft_overlong_punishment_reward/mean": -0.0397491455078125, + "rewards/soft_overlong_punishment_reward/std": 0.1816953718662262, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9902195930480957, + "sampling/importance_sampling_ratio/min": 0.2922537326812744, + "sampling/sampling_logp_difference/max": 1.230132818222046, + "sampling/sampling_logp_difference/mean": 0.01920732483267784, + "step": 181, + "step_time": 265.4475303606596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12516.0, + "completions/max_terminated_length": 12516.0, + "completions/mean_length": 8191.75, + "completions/mean_terminated_length": 8191.75, + "completions/min_length": 4992.0, + "completions/min_terminated_length": 4992.0, + "entropy": 0.26552946865558624, + "epoch": 0.2184873949579832, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4106624126434326, + "learning_rate": 2.7600000000000004e-07, + "loss": 0.0802, + "num_tokens": 62906998.0, + "reward": 2.137417793273926, + "reward_std": 2.385087013244629, + "rewards/reward_func/mean": 2.137417793273926, + "rewards/reward_func/std": 2.385087013244629, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4886276721954346, + "sampling/importance_sampling_ratio/mean": 0.9908239245414734, + "sampling/importance_sampling_ratio/min": 0.23087278008460999, + "sampling/sampling_logp_difference/max": 1.465888500213623, + "sampling/sampling_logp_difference/mean": 0.018649548292160034, + "step": 182, + "step_time": 180.85359141172376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11870.0, + "completions/max_terminated_length": 11870.0, + "completions/mean_length": 8648.4375, + "completions/mean_terminated_length": 8648.4375, + "completions/min_length": 5557.0, + "completions/min_terminated_length": 5557.0, + "entropy": 0.2885238602757454, + "epoch": 0.21968787515006002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4420488178730011, + "learning_rate": 2.72e-07, + "loss": -0.0295, + "num_tokens": 63213556.0, + "reward": 0.8226311802864075, + "reward_std": 0.6850433945655823, + "rewards/reward_func/mean": 0.8226311802864075, + "rewards/reward_func/std": 0.6850433945655823, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.508232593536377, + "sampling/importance_sampling_ratio/mean": 0.989946722984314, + "sampling/importance_sampling_ratio/min": 0.0002403110993327573, + "sampling/sampling_logp_difference/max": 8.333576202392578, + "sampling/sampling_logp_difference/mean": 0.02029946818947792, + "step": 183, + "step_time": 198.74373724497855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15138.0, + "completions/max_terminated_length": 15138.0, + "completions/mean_length": 10048.90625, + "completions/mean_terminated_length": 10048.90625, + "completions/min_length": 5452.0, + "completions/min_terminated_length": 5452.0, + "entropy": 0.24506080709397793, + "epoch": 0.22088835534213686, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3524847626686096, + "learning_rate": 2.68e-07, + "loss": 0.0155, + "num_tokens": 63572233.0, + "reward": 3.185716152191162, + "reward_std": 4.615874290466309, + "rewards/reward_func/mean": 3.185716152191162, + "rewards/reward_func/std": 4.61587381362915, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9916706085205078, + "sampling/importance_sampling_ratio/min": 0.018803074955940247, + "sampling/sampling_logp_difference/max": 3.9737348556518555, + "sampling/sampling_logp_difference/mean": 0.01703486405313015, + "step": 184, + "step_time": 224.22839038609527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11046.0, + "completions/max_terminated_length": 11046.0, + "completions/mean_length": 7622.9375, + "completions/mean_terminated_length": 7622.9375, + "completions/min_length": 4460.0, + "completions/min_terminated_length": 4460.0, + "entropy": 0.2557719862088561, + "epoch": 0.22208883553421369, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42476049065589905, + "learning_rate": 2.64e-07, + "loss": -0.0013, + "num_tokens": 63848983.0, + "reward": 4.9202985763549805, + "reward_std": 5.301763534545898, + "rewards/reward_func/mean": 4.9202985763549805, + "rewards/reward_func/std": 5.301763534545898, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9607212543487549, + "sampling/importance_sampling_ratio/mean": 0.9911612868309021, + "sampling/importance_sampling_ratio/min": 0.019983841106295586, + "sampling/sampling_logp_difference/max": 3.9128313064575195, + "sampling/sampling_logp_difference/mean": 0.017659198492765427, + "step": 185, + "step_time": 181.76806327118538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13441.0, + "completions/max_terminated_length": 13441.0, + "completions/mean_length": 9717.0, + "completions/mean_terminated_length": 9717.0, + "completions/min_length": 5815.0, + "completions/min_terminated_length": 5815.0, + "entropy": 0.24156292621046305, + "epoch": 0.22328931572629052, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.442635715007782, + "learning_rate": 2.6e-07, + "loss": -0.0102, + "num_tokens": 64210287.0, + "reward": 1.863988995552063, + "reward_std": 2.8825900554656982, + "rewards/reward_func/mean": 1.863988995552063, + "rewards/reward_func/std": 2.8825902938842773, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.648536443710327, + "sampling/importance_sampling_ratio/mean": 0.9914799928665161, + "sampling/importance_sampling_ratio/min": 0.027922935783863068, + "sampling/sampling_logp_difference/max": 3.5783069133758545, + "sampling/sampling_logp_difference/mean": 0.017204025760293007, + "step": 186, + "step_time": 238.74721054232214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13385.0, + "completions/max_terminated_length": 13385.0, + "completions/mean_length": 10068.6875, + "completions/mean_terminated_length": 10068.6875, + "completions/min_length": 7343.0, + "completions/min_terminated_length": 7343.0, + "entropy": 0.2825509188696742, + "epoch": 0.22448979591836735, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4277617335319519, + "learning_rate": 2.56e-07, + "loss": 0.0193, + "num_tokens": 64569965.0, + "reward": 0.8854573965072632, + "reward_std": 0.6112152934074402, + "rewards/reward_func/mean": 0.8854573965072632, + "rewards/reward_func/std": 0.6112152934074402, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0112133026123047, + "sampling/importance_sampling_ratio/mean": 0.9901423454284668, + "sampling/importance_sampling_ratio/min": 0.16564348340034485, + "sampling/sampling_logp_difference/max": 1.7979174852371216, + "sampling/sampling_logp_difference/mean": 0.019815031439065933, + "step": 187, + "step_time": 228.7444448468741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13373.0, + "completions/max_terminated_length": 13373.0, + "completions/mean_length": 9405.59375, + "completions/mean_terminated_length": 9405.59375, + "completions/min_length": 5967.0, + "completions/min_terminated_length": 5967.0, + "entropy": 0.318213177844882, + "epoch": 0.22569027611044418, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.33108896017074585, + "learning_rate": 2.52e-07, + "loss": 0.0167, + "num_tokens": 64897776.0, + "reward": 0.5405811071395874, + "reward_std": 0.6606644988059998, + "rewards/reward_func/mean": 0.5405811071395874, + "rewards/reward_func/std": 0.6606644988059998, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9889894723892212, + "sampling/importance_sampling_ratio/min": 0.19092479348182678, + "sampling/sampling_logp_difference/max": 1.9465055465698242, + "sampling/sampling_logp_difference/mean": 0.02206978015601635, + "step": 188, + "step_time": 415.85145999782253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11114.0, + "completions/max_terminated_length": 11114.0, + "completions/mean_length": 8580.78125, + "completions/mean_terminated_length": 8580.78125, + "completions/min_length": 4516.0, + "completions/min_terminated_length": 4516.0, + "entropy": 0.3377720918506384, + "epoch": 0.226890756302521, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42674168944358826, + "learning_rate": 2.48e-07, + "loss": 0.0088, + "num_tokens": 65198441.0, + "reward": 1.591444730758667, + "reward_std": 2.4525747299194336, + "rewards/reward_func/mean": 1.591444730758667, + "rewards/reward_func/std": 2.4525744915008545, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.6534841060638428, + "sampling/importance_sampling_ratio/mean": 0.9881213903427124, + "sampling/importance_sampling_ratio/min": 0.33775991201400757, + "sampling/sampling_logp_difference/max": 1.0854198932647705, + "sampling/sampling_logp_difference/mean": 0.022877736017107964, + "step": 189, + "step_time": 285.5526141769951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10786.0, + "completions/max_terminated_length": 10786.0, + "completions/mean_length": 7476.28125, + "completions/mean_terminated_length": 7476.28125, + "completions/min_length": 4472.0, + "completions/min_terminated_length": 4472.0, + "entropy": 0.3017116552218795, + "epoch": 0.22809123649459784, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41298624873161316, + "learning_rate": 2.4399999999999996e-07, + "loss": 0.049, + "num_tokens": 65463538.0, + "reward": 1.0010404586791992, + "reward_std": 0.7883979678153992, + "rewards/reward_func/mean": 1.0010404586791992, + "rewards/reward_func/std": 0.7883979082107544, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1348612308502197, + "sampling/importance_sampling_ratio/mean": 0.989497184753418, + "sampling/importance_sampling_ratio/min": 0.0010880796471610665, + "sampling/sampling_logp_difference/max": 6.823340892791748, + "sampling/sampling_logp_difference/mean": 0.020549368113279343, + "step": 190, + "step_time": 186.38824158336502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11019.0, + "completions/max_terminated_length": 11019.0, + "completions/mean_length": 7058.15625, + "completions/mean_terminated_length": 7058.15625, + "completions/min_length": 2793.0, + "completions/min_terminated_length": 2793.0, + "entropy": 0.29681937769055367, + "epoch": 0.22929171668667467, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.48941799998283386, + "learning_rate": 2.4e-07, + "loss": -0.0289, + "num_tokens": 65715367.0, + "reward": 2.424999475479126, + "reward_std": 2.926403045654297, + "rewards/reward_func/mean": 2.424999475479126, + "rewards/reward_func/std": 2.926403045654297, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.788496494293213, + "sampling/importance_sampling_ratio/mean": 0.9895862936973572, + "sampling/importance_sampling_ratio/min": 0.3045606315135956, + "sampling/sampling_logp_difference/max": 1.1888850927352905, + "sampling/sampling_logp_difference/mean": 0.02054070681333542, + "step": 191, + "step_time": 202.2660656850785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15425.0, + "completions/max_terminated_length": 15425.0, + "completions/mean_length": 10237.875, + "completions/mean_terminated_length": 10237.875, + "completions/min_length": 4844.0, + "completions/min_terminated_length": 4844.0, + "entropy": 0.28149267937988043, + "epoch": 0.2304921968787515, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5577535033226013, + "learning_rate": 2.3599999999999997e-07, + "loss": 0.0313, + "num_tokens": 66083091.0, + "reward": 1.852272391319275, + "reward_std": 3.534353494644165, + "rewards/reward_func/mean": 1.852272391319275, + "rewards/reward_func/std": 3.534353494644165, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9435347318649292, + "sampling/importance_sampling_ratio/mean": 0.9903722405433655, + "sampling/importance_sampling_ratio/min": 0.42083677649497986, + "sampling/sampling_logp_difference/max": 0.8655102252960205, + "sampling/sampling_logp_difference/mean": 0.019453980028629303, + "step": 192, + "step_time": 476.26090911997017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12826.0, + "completions/max_terminated_length": 12826.0, + "completions/mean_length": 9010.1875, + "completions/mean_terminated_length": 9010.1875, + "completions/min_length": 5762.0, + "completions/min_terminated_length": 5762.0, + "entropy": 0.2871004734188318, + "epoch": 0.23169267707082833, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4089401364326477, + "learning_rate": 2.32e-07, + "loss": -0.0462, + "num_tokens": 66413569.0, + "reward": 34.56924057006836, + "reward_std": 156.94326782226562, + "rewards/reward_func/mean": 34.56924057006836, + "rewards/reward_func/std": 156.9432830810547, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.8682339191436768, + "sampling/importance_sampling_ratio/mean": 0.989984929561615, + "sampling/importance_sampling_ratio/min": 0.0005362548981793225, + "sampling/sampling_logp_difference/max": 7.530900955200195, + "sampling/sampling_logp_difference/mean": 0.020170990377664566, + "step": 193, + "step_time": 259.0703094872879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10244.0, + "completions/max_terminated_length": 10244.0, + "completions/mean_length": 7021.78125, + "completions/mean_terminated_length": 7021.78125, + "completions/min_length": 4505.0, + "completions/min_terminated_length": 4505.0, + "entropy": 0.281132142059505, + "epoch": 0.23289315726290516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45947787165641785, + "learning_rate": 2.28e-07, + "loss": 0.0177, + "num_tokens": 66662410.0, + "reward": 2.6954410076141357, + "reward_std": 2.3101069927215576, + "rewards/reward_func/mean": 2.6954410076141357, + "rewards/reward_func/std": 2.3101069927215576, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2453787326812744, + "sampling/importance_sampling_ratio/mean": 0.9901114702224731, + "sampling/importance_sampling_ratio/min": 0.13286785781383514, + "sampling/sampling_logp_difference/max": 2.018400192260742, + "sampling/sampling_logp_difference/mean": 0.019676901400089264, + "step": 194, + "step_time": 158.03437204903457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 14415.0, + "completions/mean_length": 10107.21875, + "completions/mean_terminated_length": 9772.6123046875, + "completions/min_length": 5721.0, + "completions/min_terminated_length": 5721.0, + "entropy": 0.26805521361529827, + "epoch": 0.234093637454982, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3630945682525635, + "learning_rate": 2.24e-07, + "loss": -0.0655, + "num_tokens": 67019769.0, + "reward": 0.5480448007583618, + "reward_std": 0.7183238863945007, + "rewards/reward_func/mean": 0.5979817509651184, + "rewards/reward_func/std": 0.6713647246360779, + "rewards/soft_overlong_punishment_reward/mean": -0.03125, + "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 2.4275918006896973, + "sampling/importance_sampling_ratio/mean": 0.9905480146408081, + "sampling/importance_sampling_ratio/min": 0.3468351662158966, + "sampling/sampling_logp_difference/max": 1.0589056015014648, + "sampling/sampling_logp_difference/mean": 0.018695417791604996, + "step": 195, + "step_time": 256.8099763797363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11044.0, + "completions/max_terminated_length": 11044.0, + "completions/mean_length": 8487.1875, + "completions/mean_terminated_length": 8487.1875, + "completions/min_length": 4974.0, + "completions/min_terminated_length": 4974.0, + "entropy": 0.2542698457837105, + "epoch": 0.23529411764705882, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40941476821899414, + "learning_rate": 2.1999999999999998e-07, + "loss": 0.0108, + "num_tokens": 67330087.0, + "reward": 0.9295157790184021, + "reward_std": 1.449187159538269, + "rewards/reward_func/mean": 0.9295157790184021, + "rewards/reward_func/std": 1.449187159538269, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.435007333755493, + "sampling/importance_sampling_ratio/mean": 0.9911128282546997, + "sampling/importance_sampling_ratio/min": 0.0041382331401109695, + "sampling/sampling_logp_difference/max": 5.487486362457275, + "sampling/sampling_logp_difference/mean": 0.01788972318172455, + "step": 196, + "step_time": 230.38768620893825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11561.0, + "completions/max_terminated_length": 11561.0, + "completions/mean_length": 8563.40625, + "completions/mean_terminated_length": 8563.40625, + "completions/min_length": 3335.0, + "completions/min_terminated_length": 3335.0, + "entropy": 0.29917307291179895, + "epoch": 0.23649459783913565, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.4290471374988556, + "learning_rate": 2.1599999999999998e-07, + "loss": -0.0069, + "num_tokens": 67638532.0, + "reward": 0.5732239484786987, + "reward_std": 0.6618029475212097, + "rewards/reward_func/mean": 0.5732239484786987, + "rewards/reward_func/std": 0.6618028879165649, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.300884246826172, + "sampling/importance_sampling_ratio/mean": 0.9896308779716492, + "sampling/importance_sampling_ratio/min": 0.13782455027103424, + "sampling/sampling_logp_difference/max": 1.9817737340927124, + "sampling/sampling_logp_difference/mean": 0.020700525492429733, + "step": 197, + "step_time": 230.13961980782915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14545.0, + "completions/max_terminated_length": 14545.0, + "completions/mean_length": 8384.875, + "completions/mean_terminated_length": 8384.875, + "completions/min_length": 4210.0, + "completions/min_terminated_length": 4210.0, + "entropy": 0.27590967528522015, + "epoch": 0.23769507803121248, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4398875832557678, + "learning_rate": 2.12e-07, + "loss": 0.019, + "num_tokens": 67938368.0, + "reward": 29078.322265625, + "reward_std": 164479.546875, + "rewards/reward_func/mean": 29078.322265625, + "rewards/reward_func/std": 164479.546875, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.5150890350341797, + "sampling/importance_sampling_ratio/mean": 0.9903858304023743, + "sampling/importance_sampling_ratio/min": 0.07026353478431702, + "sampling/sampling_logp_difference/max": 2.6555023193359375, + "sampling/sampling_logp_difference/mean": 0.019529715180397034, + "step": 198, + "step_time": 445.3559492408531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15397.0, + "completions/max_terminated_length": 15397.0, + "completions/mean_length": 10389.4375, + "completions/mean_terminated_length": 10389.4375, + "completions/min_length": 6805.0, + "completions/min_terminated_length": 6805.0, + "entropy": 0.2911730855703354, + "epoch": 0.2388955582232893, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42178842425346375, + "learning_rate": 2.0799999999999998e-07, + "loss": 0.0221, + "num_tokens": 68309710.0, + "reward": 0.8557122945785522, + "reward_std": 0.7761005163192749, + "rewards/reward_func/mean": 0.8557122945785522, + "rewards/reward_func/std": 0.7761005759239197, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1360371112823486, + "sampling/importance_sampling_ratio/mean": 0.989779531955719, + "sampling/importance_sampling_ratio/min": 0.04221712425351143, + "sampling/sampling_logp_difference/max": 3.1649293899536133, + "sampling/sampling_logp_difference/mean": 0.02039753459393978, + "step": 199, + "step_time": 232.6083577099489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13338.0, + "completions/max_terminated_length": 13338.0, + "completions/mean_length": 9125.3125, + "completions/mean_terminated_length": 9125.3125, + "completions/min_length": 5379.0, + "completions/min_terminated_length": 5379.0, + "entropy": 0.3312236089259386, + "epoch": 0.24009603841536614, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40419086813926697, + "learning_rate": 2.0399999999999997e-07, + "loss": -0.0259, + "num_tokens": 68628320.0, + "reward": 0.9386237859725952, + "reward_std": 0.6489390730857849, + "rewards/reward_func/mean": 0.9386237859725952, + "rewards/reward_func/std": 0.6489390134811401, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.085390090942383, + "sampling/importance_sampling_ratio/mean": 0.9884665608406067, + "sampling/importance_sampling_ratio/min": 0.2759179472923279, + "sampling/sampling_logp_difference/max": 1.287651777267456, + "sampling/sampling_logp_difference/mean": 0.022430500015616417, + "step": 200, + "step_time": 201.43533559609205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11893.0, + "completions/max_terminated_length": 11893.0, + "completions/mean_length": 7656.96875, + "completions/mean_terminated_length": 7656.96875, + "completions/min_length": 3211.0, + "completions/min_terminated_length": 3211.0, + "entropy": 0.2885604901239276, + "epoch": 0.24129651860744297, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4403934180736542, + "learning_rate": 2e-07, + "loss": -0.0085, + "num_tokens": 68899535.0, + "reward": 2.592362880706787, + "reward_std": 2.72981858253479, + "rewards/reward_func/mean": 2.592362880706787, + "rewards/reward_func/std": 2.72981858253479, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9901413917541504, + "sampling/importance_sampling_ratio/min": 0.06037893146276474, + "sampling/sampling_logp_difference/max": 2.807115077972412, + "sampling/sampling_logp_difference/mean": 0.02000146359205246, + "step": 201, + "step_time": 182.84680638380814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13383.0, + "completions/max_terminated_length": 13383.0, + "completions/mean_length": 8786.125, + "completions/mean_terminated_length": 8786.125, + "completions/min_length": 4108.0, + "completions/min_terminated_length": 4108.0, + "entropy": 0.30553513020277023, + "epoch": 0.2424969987995198, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4264076352119446, + "learning_rate": 1.96e-07, + "loss": 0.0064, + "num_tokens": 69208147.0, + "reward": 1.068824291229248, + "reward_std": 0.6439973711967468, + "rewards/reward_func/mean": 1.068824291229248, + "rewards/reward_func/std": 0.643997311592102, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9403151273727417, + "sampling/importance_sampling_ratio/mean": 0.9893855452537537, + "sampling/importance_sampling_ratio/min": 0.1719096601009369, + "sampling/sampling_logp_difference/max": 1.7607861757278442, + "sampling/sampling_logp_difference/mean": 0.02125393971800804, + "step": 202, + "step_time": 202.14960491785314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13249.0, + "completions/max_terminated_length": 13249.0, + "completions/mean_length": 8494.1875, + "completions/mean_terminated_length": 8494.1875, + "completions/min_length": 2688.0, + "completions/min_terminated_length": 2688.0, + "entropy": 0.2879168214276433, + "epoch": 0.24369747899159663, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39069902896881104, + "learning_rate": 1.92e-07, + "loss": 0.0649, + "num_tokens": 69509457.0, + "reward": 1.7704880237579346, + "reward_std": 2.810868978500366, + "rewards/reward_func/mean": 1.7704880237579346, + "rewards/reward_func/std": 2.810868978500366, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9898999333381653, + "sampling/importance_sampling_ratio/min": 0.003532076720148325, + "sampling/sampling_logp_difference/max": 5.645869255065918, + "sampling/sampling_logp_difference/mean": 0.020249199122190475, + "step": 203, + "step_time": 183.80464980110992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13377.0, + "completions/max_terminated_length": 13377.0, + "completions/mean_length": 9125.28125, + "completions/mean_terminated_length": 9125.28125, + "completions/min_length": 4439.0, + "completions/min_terminated_length": 4439.0, + "entropy": 0.24132216069847345, + "epoch": 0.24489795918367346, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39876803755760193, + "learning_rate": 1.88e-07, + "loss": 0.0231, + "num_tokens": 69851202.0, + "reward": 0.5941991209983826, + "reward_std": 0.7350529432296753, + "rewards/reward_func/mean": 0.5941991209983826, + "rewards/reward_func/std": 0.7350529432296753, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.8217852115631104, + "sampling/importance_sampling_ratio/mean": 0.991719663143158, + "sampling/importance_sampling_ratio/min": 0.302060604095459, + "sampling/sampling_logp_difference/max": 1.1971275806427002, + "sampling/sampling_logp_difference/mean": 0.016849037259817123, + "step": 204, + "step_time": 229.6054396291729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12452.0, + "completions/max_terminated_length": 12452.0, + "completions/mean_length": 8173.46875, + "completions/mean_terminated_length": 8173.46875, + "completions/min_length": 4740.0, + "completions/min_terminated_length": 4740.0, + "entropy": 0.2661965722218156, + "epoch": 0.2460984393757503, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3894799053668976, + "learning_rate": 1.8399999999999998e-07, + "loss": -0.0405, + "num_tokens": 70146561.0, + "reward": 8.144660949707031, + "reward_std": 13.748440742492676, + "rewards/reward_func/mean": 8.144660949707031, + "rewards/reward_func/std": 13.748440742492676, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9909324049949646, + "sampling/importance_sampling_ratio/min": 0.17160779237747192, + "sampling/sampling_logp_difference/max": 1.7625436782836914, + "sampling/sampling_logp_difference/mean": 0.018414584919810295, + "step": 205, + "step_time": 194.27041996375192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 20480.0, + "completions/max_terminated_length": 14681.0, + "completions/mean_length": 9778.9375, + "completions/mean_terminated_length": 9433.7412109375, + "completions/min_length": 5841.0, + "completions/min_terminated_length": 5841.0, + "entropy": 0.2878849096596241, + "epoch": 0.24729891956782712, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.23984801769256592, + "learning_rate": 1.8e-07, + "loss": -0.0947, + "num_tokens": 70496111.0, + "reward": 0.5641666650772095, + "reward_std": 0.7118456363677979, + "rewards/reward_func/mean": 0.6146236658096313, + "rewards/reward_func/std": 0.6628953814506531, + "rewards/soft_overlong_punishment_reward/mean": -0.03125, + "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989820122718811, + "sampling/importance_sampling_ratio/min": 0.3761327564716339, + "sampling/sampling_logp_difference/max": 1.2785313129425049, + "sampling/sampling_logp_difference/mean": 0.020214585587382317, + "step": 206, + "step_time": 256.00012533087283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12854.0, + "completions/max_terminated_length": 12854.0, + "completions/mean_length": 7986.3125, + "completions/mean_terminated_length": 7986.3125, + "completions/min_length": 5362.0, + "completions/min_terminated_length": 5362.0, + "entropy": 0.26751279924064875, + "epoch": 0.24849939975990396, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.44527071714401245, + "learning_rate": 1.76e-07, + "loss": -0.0467, + "num_tokens": 70784649.0, + "reward": 1.1738154888153076, + "reward_std": 0.8459872603416443, + "rewards/reward_func/mean": 1.1738154888153076, + "rewards/reward_func/std": 0.8459872603416443, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.8473902940750122, + "sampling/importance_sampling_ratio/mean": 0.9906308650970459, + "sampling/importance_sampling_ratio/min": 0.3182202875614166, + "sampling/sampling_logp_difference/max": 1.1450114250183105, + "sampling/sampling_logp_difference/mean": 0.018941214308142662, + "step": 207, + "step_time": 183.06572339008562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17590.0, + "completions/max_terminated_length": 17590.0, + "completions/mean_length": 10755.09375, + "completions/mean_terminated_length": 10755.09375, + "completions/min_length": 7289.0, + "completions/min_terminated_length": 7289.0, + "entropy": 0.28774886950850487, + "epoch": 0.24969987995198079, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3857475817203522, + "learning_rate": 1.7199999999999998e-07, + "loss": 0.0468, + "num_tokens": 71168340.0, + "reward": 1.0103178024291992, + "reward_std": 1.2723941802978516, + "rewards/reward_func/mean": 1.0195188522338867, + "rewards/reward_func/std": 1.2721405029296875, + "rewards/soft_overlong_punishment_reward/mean": -0.0092010498046875, + "rewards/soft_overlong_punishment_reward/std": 0.052048999816179276, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9900601506233215, + "sampling/importance_sampling_ratio/min": 0.051096703857183456, + "sampling/sampling_logp_difference/max": 2.9740352630615234, + "sampling/sampling_logp_difference/mean": 0.01990516111254692, + "step": 208, + "step_time": 276.0685674578417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16527.0, + "completions/max_terminated_length": 16527.0, + "completions/mean_length": 10267.90625, + "completions/mean_terminated_length": 10267.90625, + "completions/min_length": 5650.0, + "completions/min_terminated_length": 5650.0, + "entropy": 0.2723705815151334, + "epoch": 0.25090036014405764, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3713222146034241, + "learning_rate": 1.68e-07, + "loss": -0.014, + "num_tokens": 71532585.0, + "reward": 1.7478203773498535, + "reward_std": 2.0512068271636963, + "rewards/reward_func/mean": 1.7489113807678223, + "rewards/reward_func/std": 2.0506913661956787, + "rewards/soft_overlong_punishment_reward/mean": -0.00109100341796875, + "rewards/soft_overlong_punishment_reward/std": 0.006171646993607283, + "sampling/importance_sampling_ratio/max": 2.043245553970337, + "sampling/importance_sampling_ratio/mean": 0.9906348586082458, + "sampling/importance_sampling_ratio/min": 0.11972002685070038, + "sampling/sampling_logp_difference/max": 2.1225993633270264, + "sampling/sampling_logp_difference/mean": 0.019026953727006912, + "step": 209, + "step_time": 249.78715924406424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14411.0, + "completions/max_terminated_length": 14411.0, + "completions/mean_length": 8568.375, + "completions/mean_terminated_length": 8568.375, + "completions/min_length": 4104.0, + "completions/min_terminated_length": 4104.0, + "entropy": 0.3192238248884678, + "epoch": 0.25210084033613445, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4139728844165802, + "learning_rate": 1.64e-07, + "loss": -0.0156, + "num_tokens": 71830213.0, + "reward": 0.5381001830101013, + "reward_std": 0.655325710773468, + "rewards/reward_func/mean": 0.5381001830101013, + "rewards/reward_func/std": 0.655325710773468, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4862608909606934, + "sampling/importance_sampling_ratio/mean": 0.9889300465583801, + "sampling/importance_sampling_ratio/min": 0.2908362150192261, + "sampling/sampling_logp_difference/max": 1.234994888305664, + "sampling/sampling_logp_difference/mean": 0.021744508296251297, + "step": 210, + "step_time": 233.44613694911823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15175.0, + "completions/max_terminated_length": 15175.0, + "completions/mean_length": 6835.25, + "completions/mean_terminated_length": 6835.25, + "completions/min_length": 2738.0, + "completions/min_terminated_length": 2738.0, + "entropy": 0.2472194992005825, + "epoch": 0.2533013205282113, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3917876183986664, + "learning_rate": 1.6e-07, + "loss": 0.0568, + "num_tokens": 72084285.0, + "reward": 2.039097785949707, + "reward_std": 2.948822498321533, + "rewards/reward_func/mean": 2.039097785949707, + "rewards/reward_func/std": 2.9488227367401123, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.354682207107544, + "sampling/importance_sampling_ratio/mean": 0.991452157497406, + "sampling/importance_sampling_ratio/min": 0.037272509187459946, + "sampling/sampling_logp_difference/max": 3.289499282836914, + "sampling/sampling_logp_difference/mean": 0.017462395131587982, + "step": 211, + "step_time": 213.77216677484103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13379.0, + "completions/max_terminated_length": 13379.0, + "completions/mean_length": 8146.84375, + "completions/mean_terminated_length": 8146.84375, + "completions/min_length": 4187.0, + "completions/min_terminated_length": 4187.0, + "entropy": 0.27302129566669464, + "epoch": 0.2545018007202881, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3473767638206482, + "learning_rate": 1.56e-07, + "loss": 0.0159, + "num_tokens": 72380368.0, + "reward": 2.9426181316375732, + "reward_std": 3.704005718231201, + "rewards/reward_func/mean": 2.9426181316375732, + "rewards/reward_func/std": 3.704005479812622, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.7764674425125122, + "sampling/importance_sampling_ratio/mean": 0.9906426668167114, + "sampling/importance_sampling_ratio/min": 0.03155454993247986, + "sampling/sampling_logp_difference/max": 3.4560375213623047, + "sampling/sampling_logp_difference/mean": 0.01900378428399563, + "step": 212, + "step_time": 210.75419173389673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14846.0, + "completions/max_terminated_length": 14846.0, + "completions/mean_length": 11225.4375, + "completions/mean_terminated_length": 11225.4375, + "completions/min_length": 7437.0, + "completions/min_terminated_length": 7437.0, + "entropy": 0.26801333110779524, + "epoch": 0.25570228091236497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.34166890382766724, + "learning_rate": 1.5199999999999998e-07, + "loss": -0.0198, + "num_tokens": 72773310.0, + "reward": 0.42110854387283325, + "reward_std": 0.5873985290527344, + "rewards/reward_func/mean": 0.42110854387283325, + "rewards/reward_func/std": 0.5873985290527344, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9907779693603516, + "sampling/importance_sampling_ratio/min": 0.34792667627334595, + "sampling/sampling_logp_difference/max": 1.1829512119293213, + "sampling/sampling_logp_difference/mean": 0.018622785806655884, + "step": 213, + "step_time": 239.31595174130052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11657.0, + "completions/max_terminated_length": 11657.0, + "completions/mean_length": 7578.28125, + "completions/mean_terminated_length": 7578.28125, + "completions/min_length": 2938.0, + "completions/min_terminated_length": 2938.0, + "entropy": 0.28475786093622446, + "epoch": 0.25690276110444177, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4476838707923889, + "learning_rate": 1.4799999999999998e-07, + "loss": -0.0071, + "num_tokens": 73049535.0, + "reward": 1.1447203159332275, + "reward_std": 1.569128155708313, + "rewards/reward_func/mean": 1.1447203159332275, + "rewards/reward_func/std": 1.569128155708313, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.6962001323699951, + "sampling/importance_sampling_ratio/mean": 0.9899388551712036, + "sampling/importance_sampling_ratio/min": 0.2515447735786438, + "sampling/sampling_logp_difference/max": 1.3801343441009521, + "sampling/sampling_logp_difference/mean": 0.019816022366285324, + "step": 214, + "step_time": 192.09097037103493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15818.0, + "completions/max_terminated_length": 15818.0, + "completions/mean_length": 9813.6875, + "completions/mean_terminated_length": 9813.6875, + "completions/min_length": 5428.0, + "completions/min_terminated_length": 5428.0, + "entropy": 0.2705380003899336, + "epoch": 0.2581032412965186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38898560404777527, + "learning_rate": 1.44e-07, + "loss": 0.0382, + "num_tokens": 73403117.0, + "reward": 0.6952791810035706, + "reward_std": 0.6618348360061646, + "rewards/reward_func/mean": 0.6952791810035706, + "rewards/reward_func/std": 0.6618348360061646, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.8969855308532715, + "sampling/importance_sampling_ratio/mean": 0.9905439615249634, + "sampling/importance_sampling_ratio/min": 0.05445502698421478, + "sampling/sampling_logp_difference/max": 2.9103801250457764, + "sampling/sampling_logp_difference/mean": 0.018809445202350616, + "step": 215, + "step_time": 219.26422131690197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12131.0, + "completions/max_terminated_length": 12131.0, + "completions/mean_length": 9191.46875, + "completions/mean_terminated_length": 9191.46875, + "completions/min_length": 4309.0, + "completions/min_terminated_length": 4309.0, + "entropy": 0.29348371736705303, + "epoch": 0.25930372148859543, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4402393102645874, + "learning_rate": 1.4e-07, + "loss": 0.0291, + "num_tokens": 73730012.0, + "reward": 2.512660264968872, + "reward_std": 2.8215131759643555, + "rewards/reward_func/mean": 2.512660264968872, + "rewards/reward_func/std": 2.8215129375457764, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989812970161438, + "sampling/importance_sampling_ratio/min": 0.16945770382881165, + "sampling/sampling_logp_difference/max": 1.7751518487930298, + "sampling/sampling_logp_difference/mean": 0.020123887807130814, + "step": 216, + "step_time": 192.42200359026901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14410.0, + "completions/max_terminated_length": 14410.0, + "completions/mean_length": 10286.53125, + "completions/mean_terminated_length": 10286.53125, + "completions/min_length": 6044.0, + "completions/min_terminated_length": 6044.0, + "entropy": 0.2660679593682289, + "epoch": 0.2605042016806723, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3704564571380615, + "learning_rate": 1.36e-07, + "loss": 0.0008, + "num_tokens": 74100413.0, + "reward": 0.5737283229827881, + "reward_std": 0.6282573342323303, + "rewards/reward_func/mean": 0.5737283229827881, + "rewards/reward_func/std": 0.6282573342323303, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.249643325805664, + "sampling/importance_sampling_ratio/mean": 0.9908007979393005, + "sampling/importance_sampling_ratio/min": 0.00029296314460225403, + "sampling/sampling_logp_difference/max": 8.13546371459961, + "sampling/sampling_logp_difference/mean": 0.018784763291478157, + "step": 217, + "step_time": 440.4557863563532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14001.0, + "completions/max_terminated_length": 14001.0, + "completions/mean_length": 10060.25, + "completions/mean_terminated_length": 10060.25, + "completions/min_length": 6824.0, + "completions/min_terminated_length": 6824.0, + "entropy": 0.3022372853010893, + "epoch": 0.2617046818727491, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.43306854367256165, + "learning_rate": 1.32e-07, + "loss": 0.0028, + "num_tokens": 74455253.0, + "reward": 16.12359619140625, + "reward_std": 59.91655731201172, + "rewards/reward_func/mean": 16.12359619140625, + "rewards/reward_func/std": 59.91655349731445, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.2559473514556885, + "sampling/importance_sampling_ratio/mean": 0.9893619418144226, + "sampling/importance_sampling_ratio/min": 0.22430028021335602, + "sampling/sampling_logp_difference/max": 1.49476957321167, + "sampling/sampling_logp_difference/mean": 0.021203139796853065, + "step": 218, + "step_time": 414.3311550068902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11622.0, + "completions/max_terminated_length": 11622.0, + "completions/mean_length": 6517.96875, + "completions/mean_terminated_length": 6517.96875, + "completions/min_length": 3112.0, + "completions/min_terminated_length": 3112.0, + "entropy": 0.28310599736869335, + "epoch": 0.26290516206482595, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.447173535823822, + "learning_rate": 1.28e-07, + "loss": -0.0289, + "num_tokens": 74686092.0, + "reward": 0.6452304720878601, + "reward_std": 2.197897434234619, + "rewards/reward_func/mean": 0.6452304720878601, + "rewards/reward_func/std": 2.197897434234619, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.134779691696167, + "sampling/importance_sampling_ratio/mean": 0.9900719523429871, + "sampling/importance_sampling_ratio/min": 0.36751604080200195, + "sampling/sampling_logp_difference/max": 1.0009883642196655, + "sampling/sampling_logp_difference/mean": 0.0198367852717638, + "step": 219, + "step_time": 177.24385808466468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11380.0, + "completions/max_terminated_length": 11380.0, + "completions/mean_length": 6471.75, + "completions/mean_terminated_length": 6471.75, + "completions/min_length": 4281.0, + "completions/min_terminated_length": 4281.0, + "entropy": 0.3177992310374975, + "epoch": 0.26410564225690275, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5168129205703735, + "learning_rate": 1.24e-07, + "loss": 0.0395, + "num_tokens": 74912764.0, + "reward": 2.1183838844299316, + "reward_std": 1.930953025817871, + "rewards/reward_func/mean": 2.1183838844299316, + "rewards/reward_func/std": 1.930953025817871, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0708396434783936, + "sampling/importance_sampling_ratio/mean": 0.9886492490768433, + "sampling/importance_sampling_ratio/min": 0.3355446755886078, + "sampling/sampling_logp_difference/max": 1.0920002460479736, + "sampling/sampling_logp_difference/mean": 0.022046584635972977, + "step": 220, + "step_time": 158.16589547123294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14608.0, + "completions/max_terminated_length": 14608.0, + "completions/mean_length": 9121.625, + "completions/mean_terminated_length": 9121.625, + "completions/min_length": 2448.0, + "completions/min_terminated_length": 2448.0, + "entropy": 0.2602370521053672, + "epoch": 0.2653061224489796, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.39733272790908813, + "learning_rate": 1.2e-07, + "loss": 0.0234, + "num_tokens": 75242344.0, + "reward": 0.6069552898406982, + "reward_std": 0.6848835945129395, + "rewards/reward_func/mean": 0.6069552898406982, + "rewards/reward_func/std": 0.6848835349082947, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.775014877319336, + "sampling/importance_sampling_ratio/mean": 0.9909803867340088, + "sampling/importance_sampling_ratio/min": 0.14724299311637878, + "sampling/sampling_logp_difference/max": 1.9156711101531982, + "sampling/sampling_logp_difference/mean": 0.01835060492157936, + "step": 221, + "step_time": 228.61106476001441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12311.0, + "completions/max_terminated_length": 12311.0, + "completions/mean_length": 8921.34375, + "completions/mean_terminated_length": 8921.34375, + "completions/min_length": 5336.0, + "completions/min_terminated_length": 5336.0, + "entropy": 0.2625180771574378, + "epoch": 0.2665066026410564, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3746626079082489, + "learning_rate": 1.16e-07, + "loss": -0.0205, + "num_tokens": 75562803.0, + "reward": 1.2791639566421509, + "reward_std": 1.4882169961929321, + "rewards/reward_func/mean": 1.2791639566421509, + "rewards/reward_func/std": 1.4882171154022217, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9303572177886963, + "sampling/importance_sampling_ratio/mean": 0.9908260107040405, + "sampling/importance_sampling_ratio/min": 0.32418113946914673, + "sampling/sampling_logp_difference/max": 1.1264528036117554, + "sampling/sampling_logp_difference/mean": 0.01839890144765377, + "step": 222, + "step_time": 331.951701607788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15993.0, + "completions/max_terminated_length": 15993.0, + "completions/mean_length": 9919.09375, + "completions/mean_terminated_length": 9919.09375, + "completions/min_length": 2850.0, + "completions/min_terminated_length": 2850.0, + "entropy": 0.27178078424185514, + "epoch": 0.26770708283313327, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.32960885763168335, + "learning_rate": 1.12e-07, + "loss": -0.0391, + "num_tokens": 75917510.0, + "reward": 0.9057698845863342, + "reward_std": 1.4171531200408936, + "rewards/reward_func/mean": 0.9057698845863342, + "rewards/reward_func/std": 1.4171531200408936, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9905833005905151, + "sampling/importance_sampling_ratio/min": 0.12444481998682022, + "sampling/sampling_logp_difference/max": 2.083892822265625, + "sampling/sampling_logp_difference/mean": 0.019010471180081367, + "step": 223, + "step_time": 452.7820453970926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15207.0, + "completions/max_terminated_length": 15207.0, + "completions/mean_length": 8989.75, + "completions/mean_terminated_length": 8989.75, + "completions/min_length": 5085.0, + "completions/min_terminated_length": 5085.0, + "entropy": 0.27119907177984715, + "epoch": 0.2689075630252101, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3286755681037903, + "learning_rate": 1.0799999999999999e-07, + "loss": -0.01, + "num_tokens": 76241470.0, + "reward": 2.0997040271759033, + "reward_std": 2.725891351699829, + "rewards/reward_func/mean": 2.0997040271759033, + "rewards/reward_func/std": 2.72589111328125, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.770100474357605, + "sampling/importance_sampling_ratio/mean": 0.9906846284866333, + "sampling/importance_sampling_ratio/min": 0.03444494679570198, + "sampling/sampling_logp_difference/max": 3.3683929443359375, + "sampling/sampling_logp_difference/mean": 0.018905045464634895, + "step": 224, + "step_time": 434.58480245189276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14341.0, + "completions/max_terminated_length": 14341.0, + "completions/mean_length": 9169.65625, + "completions/mean_terminated_length": 9169.65625, + "completions/min_length": 4655.0, + "completions/min_terminated_length": 4655.0, + "entropy": 0.2545336727052927, + "epoch": 0.27010804321728693, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.43626582622528076, + "learning_rate": 1.0399999999999999e-07, + "loss": -0.0511, + "num_tokens": 76579227.0, + "reward": 0.7725225687026978, + "reward_std": 0.8143731951713562, + "rewards/reward_func/mean": 0.7725225687026978, + "rewards/reward_func/std": 0.8143731951713562, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.39496111869812, + "sampling/importance_sampling_ratio/mean": 0.9909839630126953, + "sampling/importance_sampling_ratio/min": 0.17538554966449738, + "sampling/sampling_logp_difference/max": 1.7407686710357666, + "sampling/sampling_logp_difference/mean": 0.018118245527148247, + "step": 225, + "step_time": 256.96223465609364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11969.0, + "completions/max_terminated_length": 11969.0, + "completions/mean_length": 7255.71875, + "completions/mean_terminated_length": 7255.71875, + "completions/min_length": 3750.0, + "completions/min_terminated_length": 3750.0, + "entropy": 0.2862818920984864, + "epoch": 0.27130852340936373, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5091444253921509, + "learning_rate": 1e-07, + "loss": 0.0555, + "num_tokens": 76838802.0, + "reward": 3.338810443878174, + "reward_std": 3.9317619800567627, + "rewards/reward_func/mean": 3.338810443878174, + "rewards/reward_func/std": 3.9317619800567627, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1975717544555664, + "sampling/importance_sampling_ratio/mean": 0.9902316331863403, + "sampling/importance_sampling_ratio/min": 0.10265523940324783, + "sampling/sampling_logp_difference/max": 2.276379108428955, + "sampling/sampling_logp_difference/mean": 0.01985924504697323, + "step": 226, + "step_time": 177.48551015171688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14337.0, + "completions/max_terminated_length": 14337.0, + "completions/mean_length": 8913.25, + "completions/mean_terminated_length": 8913.25, + "completions/min_length": 3138.0, + "completions/min_terminated_length": 3138.0, + "entropy": 0.26585716381669044, + "epoch": 0.2725090036014406, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3888947069644928, + "learning_rate": 9.6e-08, + "loss": -0.0538, + "num_tokens": 77161562.0, + "reward": 2.5214858055114746, + "reward_std": 2.8809354305267334, + "rewards/reward_func/mean": 2.5214858055114746, + "rewards/reward_func/std": 2.8809354305267334, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.9848451614379883, + "sampling/importance_sampling_ratio/mean": 0.9906054139137268, + "sampling/importance_sampling_ratio/min": 0.12535132467746735, + "sampling/sampling_logp_difference/max": 2.0766348838806152, + "sampling/sampling_logp_difference/mean": 0.01874399371445179, + "step": 227, + "step_time": 423.3902858378133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12350.0, + "completions/max_terminated_length": 12350.0, + "completions/mean_length": 9451.5625, + "completions/mean_terminated_length": 9451.5625, + "completions/min_length": 6397.0, + "completions/min_terminated_length": 6397.0, + "entropy": 0.2693768171593547, + "epoch": 0.2737094837935174, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3820766508579254, + "learning_rate": 9.199999999999999e-08, + "loss": 0.033, + "num_tokens": 77498220.0, + "reward": 1.154233455657959, + "reward_std": 1.959659218788147, + "rewards/reward_func/mean": 1.154233455657959, + "rewards/reward_func/std": 1.959659218788147, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9909024238586426, + "sampling/importance_sampling_ratio/min": 0.20406676828861237, + "sampling/sampling_logp_difference/max": 1.5893080234527588, + "sampling/sampling_logp_difference/mean": 0.018469782546162605, + "step": 228, + "step_time": 217.56979901902378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12107.0, + "completions/max_terminated_length": 12107.0, + "completions/mean_length": 8378.96875, + "completions/mean_terminated_length": 8378.96875, + "completions/min_length": 5003.0, + "completions/min_terminated_length": 5003.0, + "entropy": 0.2748628966510296, + "epoch": 0.27490996398559425, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4501209557056427, + "learning_rate": 8.8e-08, + "loss": -0.0042, + "num_tokens": 77802059.0, + "reward": 1.3091723918914795, + "reward_std": 0.47337406873703003, + "rewards/reward_func/mean": 1.3091723918914795, + "rewards/reward_func/std": 0.47337406873703003, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0756125450134277, + "sampling/importance_sampling_ratio/mean": 0.9905352592468262, + "sampling/importance_sampling_ratio/min": 0.22848102450370789, + "sampling/sampling_logp_difference/max": 1.476302146911621, + "sampling/sampling_logp_difference/mean": 0.01908484473824501, + "step": 229, + "step_time": 173.2410452858312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11253.0, + "completions/max_terminated_length": 11253.0, + "completions/mean_length": 8427.15625, + "completions/mean_terminated_length": 8427.15625, + "completions/min_length": 5975.0, + "completions/min_terminated_length": 5975.0, + "entropy": 0.28478993009775877, + "epoch": 0.27611044417767105, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42257431149482727, + "learning_rate": 8.4e-08, + "loss": 0.0338, + "num_tokens": 78102984.0, + "reward": 0.6847608089447021, + "reward_std": 0.6159871220588684, + "rewards/reward_func/mean": 0.6847608089447021, + "rewards/reward_func/std": 0.6159871220588684, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.810465693473816, + "sampling/importance_sampling_ratio/mean": 0.9901672601699829, + "sampling/importance_sampling_ratio/min": 0.21681459248065948, + "sampling/sampling_logp_difference/max": 1.5287127494812012, + "sampling/sampling_logp_difference/mean": 0.01989746280014515, + "step": 230, + "step_time": 376.1443422808079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13502.0, + "completions/max_terminated_length": 13502.0, + "completions/mean_length": 9104.8125, + "completions/mean_terminated_length": 9104.8125, + "completions/min_length": 5037.0, + "completions/min_terminated_length": 5037.0, + "entropy": 0.28373472206294537, + "epoch": 0.2773109243697479, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3297131657600403, + "learning_rate": 8e-08, + "loss": -0.004, + "num_tokens": 78423906.0, + "reward": 3.833698272705078, + "reward_std": 8.968595504760742, + "rewards/reward_func/mean": 3.833698272705078, + "rewards/reward_func/std": 8.968595504760742, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.7473583221435547, + "sampling/importance_sampling_ratio/mean": 0.9901688694953918, + "sampling/importance_sampling_ratio/min": 0.3091135621070862, + "sampling/sampling_logp_difference/max": 1.174046516418457, + "sampling/sampling_logp_difference/mean": 0.019583595916628838, + "step": 231, + "step_time": 192.75546575232875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13344.0, + "completions/max_terminated_length": 13344.0, + "completions/mean_length": 9607.15625, + "completions/mean_terminated_length": 9607.15625, + "completions/min_length": 6523.0, + "completions/min_terminated_length": 6523.0, + "entropy": 0.258233230561018, + "epoch": 0.2785114045618247, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.22545225918293, + "learning_rate": 7.599999999999999e-08, + "loss": 0.0098, + "num_tokens": 78774783.0, + "reward": 1.9562370777130127, + "reward_std": 3.413482666015625, + "rewards/reward_func/mean": 1.9562370777130127, + "rewards/reward_func/std": 3.413482666015625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9909834861755371, + "sampling/importance_sampling_ratio/min": 0.03608707711100578, + "sampling/sampling_logp_difference/max": 3.3218204975128174, + "sampling/sampling_logp_difference/mean": 0.01814298704266548, + "step": 232, + "step_time": 441.14181373117026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14557.0, + "completions/max_terminated_length": 14557.0, + "completions/mean_length": 8880.46875, + "completions/mean_terminated_length": 8880.46875, + "completions/min_length": 3949.0, + "completions/min_terminated_length": 3949.0, + "entropy": 0.2796018850058317, + "epoch": 0.2797118847539016, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.41359373927116394, + "learning_rate": 7.2e-08, + "loss": 0.0204, + "num_tokens": 79094742.0, + "reward": 2.0865750312805176, + "reward_std": 3.248898506164551, + "rewards/reward_func/mean": 2.0865750312805176, + "rewards/reward_func/std": 3.248898506164551, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.355583906173706, + "sampling/importance_sampling_ratio/mean": 0.9904192090034485, + "sampling/importance_sampling_ratio/min": 0.22791339457035065, + "sampling/sampling_logp_difference/max": 1.4787895679473877, + "sampling/sampling_logp_difference/mean": 0.019396543502807617, + "step": 233, + "step_time": 433.1994162177434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13023.0, + "completions/max_terminated_length": 13023.0, + "completions/mean_length": 9278.125, + "completions/mean_terminated_length": 9278.125, + "completions/min_length": 6046.0, + "completions/min_terminated_length": 6046.0, + "entropy": 0.23018638603389263, + "epoch": 0.2809123649459784, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40194013714790344, + "learning_rate": 6.8e-08, + "loss": -0.0207, + "num_tokens": 79432034.0, + "reward": 1.2649976015090942, + "reward_std": 2.4887611865997314, + "rewards/reward_func/mean": 1.2649976015090942, + "rewards/reward_func/std": 2.4887611865997314, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.066953420639038, + "sampling/importance_sampling_ratio/mean": 0.9919387698173523, + "sampling/importance_sampling_ratio/min": 0.07596759498119354, + "sampling/sampling_logp_difference/max": 2.5774483680725098, + "sampling/sampling_logp_difference/mean": 0.01637454889714718, + "step": 234, + "step_time": 200.44299350422807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11484.0, + "completions/max_terminated_length": 11484.0, + "completions/mean_length": 7951.46875, + "completions/mean_terminated_length": 7951.46875, + "completions/min_length": 4951.0, + "completions/min_terminated_length": 4951.0, + "entropy": 0.28384537622332573, + "epoch": 0.28211284513805523, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4639303684234619, + "learning_rate": 6.4e-08, + "loss": 0.0062, + "num_tokens": 79715057.0, + "reward": 1.722654938697815, + "reward_std": 2.3874762058258057, + "rewards/reward_func/mean": 1.722654938697815, + "rewards/reward_func/std": 2.3874759674072266, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9900355339050293, + "sampling/importance_sampling_ratio/min": 0.16387537121772766, + "sampling/sampling_logp_difference/max": 1.8086490631103516, + "sampling/sampling_logp_difference/mean": 0.019974499940872192, + "step": 235, + "step_time": 188.83989403117448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15359.0, + "completions/max_terminated_length": 15359.0, + "completions/mean_length": 9800.75, + "completions/mean_terminated_length": 9800.75, + "completions/min_length": 6034.0, + "completions/min_terminated_length": 6034.0, + "entropy": 0.3060651607811451, + "epoch": 0.28331332533013204, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.4038175940513611, + "learning_rate": 6e-08, + "loss": 0.0295, + "num_tokens": 80063393.0, + "reward": 0.5499008893966675, + "reward_std": 0.9594078660011292, + "rewards/reward_func/mean": 0.5499008893966675, + "rewards/reward_func/std": 0.9594078660011292, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.989382803440094, + "sampling/importance_sampling_ratio/min": 0.10317613184452057, + "sampling/sampling_logp_difference/max": 2.271317720413208, + "sampling/sampling_logp_difference/mean": 0.021366208791732788, + "step": 236, + "step_time": 246.472208352061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14811.0, + "completions/max_terminated_length": 14811.0, + "completions/mean_length": 10420.625, + "completions/mean_terminated_length": 10420.625, + "completions/min_length": 7528.0, + "completions/min_terminated_length": 7528.0, + "entropy": 0.26841907389461994, + "epoch": 0.2845138055222089, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3900371789932251, + "learning_rate": 5.6e-08, + "loss": 0.0085, + "num_tokens": 80437933.0, + "reward": 0.7362344264984131, + "reward_std": 0.9423819780349731, + "rewards/reward_func/mean": 0.7362344264984131, + "rewards/reward_func/std": 0.9423819184303284, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9905475378036499, + "sampling/importance_sampling_ratio/min": 0.06430970132350922, + "sampling/sampling_logp_difference/max": 2.744044780731201, + "sampling/sampling_logp_difference/mean": 0.018762534484267235, + "step": 237, + "step_time": 224.29740512394346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19168.0, + "completions/max_terminated_length": 19168.0, + "completions/mean_length": 10909.96875, + "completions/mean_terminated_length": 10909.96875, + "completions/min_length": 7484.0, + "completions/min_terminated_length": 7484.0, + "entropy": 0.2649214630946517, + "epoch": 0.2857142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3601086735725403, + "learning_rate": 5.1999999999999996e-08, + "loss": 0.0136, + "num_tokens": 80832676.0, + "reward": 1.2694827318191528, + "reward_std": 1.707560420036316, + "rewards/reward_func/mean": 1.2907229661941528, + "rewards/reward_func/std": 1.7039649486541748, + "rewards/soft_overlong_punishment_reward/mean": -0.021240234375, + "rewards/soft_overlong_punishment_reward/std": 0.12015291303396225, + "sampling/importance_sampling_ratio/max": 2.5833005905151367, + "sampling/importance_sampling_ratio/mean": 0.990875244140625, + "sampling/importance_sampling_ratio/min": 0.3051411211490631, + "sampling/sampling_logp_difference/max": 1.186980962753296, + "sampling/sampling_logp_difference/mean": 0.018429221585392952, + "step": 238, + "step_time": 254.5579416519031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13040.0, + "completions/max_terminated_length": 13040.0, + "completions/mean_length": 7543.53125, + "completions/mean_terminated_length": 7543.53125, + "completions/min_length": 4494.0, + "completions/min_terminated_length": 4494.0, + "entropy": 0.2839345559477806, + "epoch": 0.28691476590636256, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.487249493598938, + "learning_rate": 4.8e-08, + "loss": 0.028, + "num_tokens": 81106021.0, + "reward": 0.7453573942184448, + "reward_std": 0.7949322462081909, + "rewards/reward_func/mean": 0.7453573942184448, + "rewards/reward_func/std": 0.7949321269989014, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.3466830253601074, + "sampling/importance_sampling_ratio/mean": 0.9900220036506653, + "sampling/importance_sampling_ratio/min": 0.012635735794901848, + "sampling/sampling_logp_difference/max": 4.3712263107299805, + "sampling/sampling_logp_difference/mean": 0.019755955785512924, + "step": 239, + "step_time": 204.47343361668754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13324.0, + "completions/max_terminated_length": 13324.0, + "completions/mean_length": 8653.3125, + "completions/mean_terminated_length": 8653.3125, + "completions/min_length": 4673.0, + "completions/min_terminated_length": 4673.0, + "entropy": 0.2574413791298866, + "epoch": 0.28811524609843936, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3604678213596344, + "learning_rate": 4.4e-08, + "loss": -0.0105, + "num_tokens": 81421999.0, + "reward": 1.671258568763733, + "reward_std": 2.822765350341797, + "rewards/reward_func/mean": 1.671258568763733, + "rewards/reward_func/std": 2.822765588760376, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.407453775405884, + "sampling/importance_sampling_ratio/mean": 0.9911367893218994, + "sampling/importance_sampling_ratio/min": 0.35551854968070984, + "sampling/sampling_logp_difference/max": 1.0341778993606567, + "sampling/sampling_logp_difference/mean": 0.018147394061088562, + "step": 240, + "step_time": 216.4728114741156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 15334.0, + "completions/max_terminated_length": 15334.0, + "completions/mean_length": 10050.375, + "completions/mean_terminated_length": 10050.375, + "completions/min_length": 3177.0, + "completions/min_terminated_length": 3177.0, + "entropy": 0.25266273971647024, + "epoch": 0.2893157262905162, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.34388092160224915, + "learning_rate": 4e-08, + "loss": -0.0302, + "num_tokens": 81783875.0, + "reward": 2.149714469909668, + "reward_std": 2.923807144165039, + "rewards/reward_func/mean": 2.149714469909668, + "rewards/reward_func/std": 2.92380690574646, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9911876916885376, + "sampling/importance_sampling_ratio/min": 0.00033268670085817575, + "sampling/sampling_logp_difference/max": 8.008309364318848, + "sampling/sampling_logp_difference/mean": 0.01783919706940651, + "step": 241, + "step_time": 258.6670099237235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 10994.0, + "completions/max_terminated_length": 10994.0, + "completions/mean_length": 8268.15625, + "completions/mean_terminated_length": 8268.15625, + "completions/min_length": 5467.0, + "completions/min_terminated_length": 5467.0, + "entropy": 0.2947918279096484, + "epoch": 0.290516206482593, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.37617602944374084, + "learning_rate": 3.6e-08, + "loss": 0.045, + "num_tokens": 82076200.0, + "reward": 1.3230187892913818, + "reward_std": 1.936329960823059, + "rewards/reward_func/mean": 1.3230187892913818, + "rewards/reward_func/std": 1.9363298416137695, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.1945412158966064, + "sampling/importance_sampling_ratio/mean": 0.98971027135849, + "sampling/importance_sampling_ratio/min": 0.0968865156173706, + "sampling/sampling_logp_difference/max": 2.334214925765991, + "sampling/sampling_logp_difference/mean": 0.020341265946626663, + "step": 242, + "step_time": 192.7506662090309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12898.0, + "completions/max_terminated_length": 12898.0, + "completions/mean_length": 9256.78125, + "completions/mean_terminated_length": 9256.78125, + "completions/min_length": 4686.0, + "completions/min_terminated_length": 4686.0, + "entropy": 0.30538202077150345, + "epoch": 0.2917166866746699, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.3372436761856079, + "learning_rate": 3.2e-08, + "loss": 0.0148, + "num_tokens": 82395961.0, + "reward": 0.8563218712806702, + "reward_std": 1.9926910400390625, + "rewards/reward_func/mean": 0.8563218712806702, + "rewards/reward_func/std": 1.9926910400390625, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9894178509712219, + "sampling/importance_sampling_ratio/min": 0.13205546140670776, + "sampling/sampling_logp_difference/max": 2.024533271789551, + "sampling/sampling_logp_difference/mean": 0.02108946070075035, + "step": 243, + "step_time": 222.2967141327681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 13215.0, + "completions/max_terminated_length": 13215.0, + "completions/mean_length": 9513.03125, + "completions/mean_terminated_length": 9513.03125, + "completions/min_length": 6086.0, + "completions/min_terminated_length": 6086.0, + "entropy": 0.26438521407544613, + "epoch": 0.2929171668667467, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40192142128944397, + "learning_rate": 2.8e-08, + "loss": 0.0154, + "num_tokens": 82738746.0, + "reward": 3.0766522884368896, + "reward_std": 2.7076849937438965, + "rewards/reward_func/mean": 3.0766522884368896, + "rewards/reward_func/std": 2.7076849937438965, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9906589388847351, + "sampling/importance_sampling_ratio/min": 0.27606433629989624, + "sampling/sampling_logp_difference/max": 1.287121295928955, + "sampling/sampling_logp_difference/mean": 0.018659161403775215, + "step": 244, + "step_time": 206.91322167252656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 17517.0, + "completions/max_terminated_length": 17517.0, + "completions/mean_length": 9881.0625, + "completions/mean_terminated_length": 9881.0625, + "completions/min_length": 2381.0, + "completions/min_terminated_length": 2381.0, + "entropy": 0.25762535352259874, + "epoch": 0.29411764705882354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.40251660346984863, + "learning_rate": 2.4e-08, + "loss": 0.0311, + "num_tokens": 83097732.0, + "reward": 3.15523099899292, + "reward_std": 5.307023525238037, + "rewards/reward_func/mean": 3.163875102996826, + "rewards/reward_func/std": 5.301476001739502, + "rewards/soft_overlong_punishment_reward/mean": -0.00864410400390625, + "rewards/soft_overlong_punishment_reward/std": 0.04889843612909317, + "sampling/importance_sampling_ratio/max": 3.0, + "sampling/importance_sampling_ratio/mean": 0.9913111925125122, + "sampling/importance_sampling_ratio/min": 0.0033161414321511984, + "sampling/sampling_logp_difference/max": 5.708953380584717, + "sampling/sampling_logp_difference/mean": 0.01810595765709877, + "step": 245, + "step_time": 277.0211471380899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 16853.0, + "completions/max_terminated_length": 16853.0, + "completions/mean_length": 10218.5, + "completions/mean_terminated_length": 10218.5, + "completions/min_length": 6606.0, + "completions/min_terminated_length": 6606.0, + "entropy": 0.28045162558555603, + "epoch": 0.29531812725090034, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3734090030193329, + "learning_rate": 2e-08, + "loss": 0.001, + "num_tokens": 83459092.0, + "reward": 0.8230442404747009, + "reward_std": 0.6862863898277283, + "rewards/reward_func/mean": 0.8266224265098572, + "rewards/reward_func/std": 0.6817933917045593, + "rewards/soft_overlong_punishment_reward/mean": -0.00357818603515625, + "rewards/soft_overlong_punishment_reward/std": 0.020241277292370796, + "sampling/importance_sampling_ratio/max": 2.0942885875701904, + "sampling/importance_sampling_ratio/mean": 0.990305483341217, + "sampling/importance_sampling_ratio/min": 0.018712347373366356, + "sampling/sampling_logp_difference/max": 3.978571653366089, + "sampling/sampling_logp_difference/mean": 0.019516419619321823, + "step": 246, + "step_time": 472.73078032024205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 19484.0, + "completions/max_terminated_length": 19484.0, + "completions/mean_length": 10279.5625, + "completions/mean_terminated_length": 10279.5625, + "completions/min_length": 5641.0, + "completions/min_terminated_length": 5641.0, + "entropy": 0.2904065754264593, + "epoch": 0.2965186074429772, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.3862820565700531, + "learning_rate": 1.6e-08, + "loss": 0.032, + "num_tokens": 83814838.0, + "reward": 1.4464482069015503, + "reward_std": 2.287731885910034, + "rewards/reward_func/mean": 1.4815586805343628, + "rewards/reward_func/std": 2.2599167823791504, + "rewards/soft_overlong_punishment_reward/mean": -0.0351104736328125, + "rewards/soft_overlong_punishment_reward/std": 0.146774023771286, + "sampling/importance_sampling_ratio/max": 2.4637815952301025, + "sampling/importance_sampling_ratio/mean": 0.989745557308197, + "sampling/importance_sampling_ratio/min": 0.15747244656085968, + "sampling/sampling_logp_difference/max": 1.8485047817230225, + "sampling/sampling_logp_difference/mean": 0.020400259643793106, + "step": 247, + "step_time": 332.2391231590882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 12802.0, + "completions/max_terminated_length": 12802.0, + "completions/mean_length": 7361.375, + "completions/mean_terminated_length": 7361.375, + "completions/min_length": 5259.0, + "completions/min_terminated_length": 5259.0, + "entropy": 0.3144523575901985, + "epoch": 0.297719087635054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45082876086235046, + "learning_rate": 1.2e-08, + "loss": 0.0151, + "num_tokens": 84073074.0, + "reward": 1.048694372177124, + "reward_std": 0.5158096551895142, + "rewards/reward_func/mean": 1.048694372177124, + "rewards/reward_func/std": 0.5158095955848694, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.7785451412200928, + "sampling/importance_sampling_ratio/mean": 0.9890033602714539, + "sampling/importance_sampling_ratio/min": 0.31890588998794556, + "sampling/sampling_logp_difference/max": 1.1428592205047607, + "sampling/sampling_logp_difference/mean": 0.02159862592816353, + "step": 248, + "step_time": 215.50420774670783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 14278.0, + "completions/max_terminated_length": 14278.0, + "completions/mean_length": 9695.75, + "completions/mean_terminated_length": 9695.75, + "completions/min_length": 4891.0, + "completions/min_terminated_length": 4891.0, + "entropy": 0.2848109360784292, + "epoch": 0.29891956782713086, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.38538479804992676, + "learning_rate": 8e-09, + "loss": -0.0234, + "num_tokens": 84425578.0, + "reward": 1.7004798650741577, + "reward_std": 2.567974805831909, + "rewards/reward_func/mean": 1.7004798650741577, + "rewards/reward_func/std": 2.567974805831909, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.873475432395935, + "sampling/importance_sampling_ratio/mean": 0.9899736642837524, + "sampling/importance_sampling_ratio/min": 0.29256924986839294, + "sampling/sampling_logp_difference/max": 1.2290538549423218, + "sampling/sampling_logp_difference/mean": 0.01992029696702957, + "step": 249, + "step_time": 247.963375205989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 11532.0, + "completions/max_terminated_length": 11532.0, + "completions/mean_length": 9036.21875, + "completions/mean_terminated_length": 9036.21875, + "completions/min_length": 4886.0, + "completions/min_terminated_length": 4886.0, + "entropy": 0.27648542169481516, + "epoch": 0.30012004801920766, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8751509189605713, + "learning_rate": 4e-09, + "loss": -0.0151, + "num_tokens": 84746073.0, + "reward": 3.147611618041992, + "reward_std": 4.971174716949463, + "rewards/reward_func/mean": 3.147611618041992, + "rewards/reward_func/std": 4.971174716949463, + "rewards/soft_overlong_punishment_reward/mean": 0.0, + "rewards/soft_overlong_punishment_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.4234373569488525, + "sampling/importance_sampling_ratio/mean": 0.9902631640434265, + "sampling/importance_sampling_ratio/min": 0.09392388164997101, + "sampling/sampling_logp_difference/max": 2.3652706146240234, + "sampling/sampling_logp_difference/mean": 0.019402729347348213, + "step": 250, + "step_time": 203.76658588682767 + } + ], + "logging_steps": 1, + "max_steps": 250, + "num_input_tokens_seen": 84746073, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}