| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.30012004801920766, |
| "eval_steps": 500, |
| "global_step": 250, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17003.0, |
| "completions/max_terminated_length": 17003.0, |
| "completions/mean_length": 10512.40625, |
| "completions/mean_terminated_length": 10512.40625, |
| "completions/min_length": 6733.0, |
| "completions/min_terminated_length": 6733.0, |
| "entropy": 0.2287780987098813, |
| "epoch": 0.0012004801920768306, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3057364225387573, |
| "learning_rate": 1e-06, |
| "loss": 0.0187, |
| "num_tokens": 377149.0, |
| "reward": 2.172255277633667, |
| "reward_std": 2.8551955223083496, |
| "rewards/reward_func/mean": 2.1769778728485107, |
| "rewards/reward_func/std": 2.863737106323242, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00472259521484375, |
| "rewards/soft_overlong_punishment_reward/std": 0.026715034618973732, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9920079708099365, |
| "sampling/importance_sampling_ratio/min": 0.06831115484237671, |
| "sampling/sampling_logp_difference/max": 2.6836822032928467, |
| "sampling/sampling_logp_difference/mean": 0.016660798341035843, |
| "step": 1, |
| "step_time": 260.7638783827424 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13544.0, |
| "completions/max_terminated_length": 13544.0, |
| "completions/mean_length": 9758.78125, |
| "completions/mean_terminated_length": 9758.78125, |
| "completions/min_length": 7933.0, |
| "completions/min_terminated_length": 7933.0, |
| "entropy": 0.26108355820178986, |
| "epoch": 0.0024009603841536613, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.45485132932662964, |
| "learning_rate": 9.959999999999999e-07, |
| "loss": -0.0342, |
| "num_tokens": 724854.0, |
| "reward": 2.3083786964416504, |
| "reward_std": 3.1393442153930664, |
| "rewards/reward_func/mean": 2.3083786964416504, |
| "rewards/reward_func/std": 3.1393442153930664, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9068528413772583, |
| "sampling/importance_sampling_ratio/mean": 0.9908677339553833, |
| "sampling/importance_sampling_ratio/min": 0.009255149401724339, |
| "sampling/sampling_logp_difference/max": 4.682575225830078, |
| "sampling/sampling_logp_difference/mean": 0.01871836744248867, |
| "step": 2, |
| "step_time": 251.4262609881116 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14033.0, |
| "completions/max_terminated_length": 14033.0, |
| "completions/mean_length": 9934.3125, |
| "completions/mean_terminated_length": 9934.3125, |
| "completions/min_length": 4489.0, |
| "completions/min_terminated_length": 4489.0, |
| "entropy": 0.26445347908884287, |
| "epoch": 0.003601440576230492, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3962293267250061, |
| "learning_rate": 9.92e-07, |
| "loss": -0.0314, |
| "num_tokens": 1075280.0, |
| "reward": 2.6239943504333496, |
| "reward_std": 2.9723362922668457, |
| "rewards/reward_func/mean": 2.6239943504333496, |
| "rewards/reward_func/std": 2.972336769104004, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1135287284851074, |
| "sampling/importance_sampling_ratio/mean": 0.9906726479530334, |
| "sampling/importance_sampling_ratio/min": 0.20889143645763397, |
| "sampling/sampling_logp_difference/max": 1.5659406185150146, |
| "sampling/sampling_logp_difference/mean": 0.01895604282617569, |
| "step": 3, |
| "step_time": 210.0828171017347 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15270.0, |
| "completions/max_terminated_length": 15270.0, |
| "completions/mean_length": 10381.21875, |
| "completions/mean_terminated_length": 10381.21875, |
| "completions/min_length": 5544.0, |
| "completions/min_terminated_length": 5544.0, |
| "entropy": 0.2782252887263894, |
| "epoch": 0.004801920768307323, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.33767879009246826, |
| "learning_rate": 9.88e-07, |
| "loss": -0.0169, |
| "num_tokens": 1430327.0, |
| "reward": 1775.4732666015625, |
| "reward_std": 4829.59228515625, |
| "rewards/reward_func/mean": 1775.4732666015625, |
| "rewards/reward_func/std": 4829.5927734375, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.093236207962036, |
| "sampling/importance_sampling_ratio/mean": 0.9902609586715698, |
| "sampling/importance_sampling_ratio/min": 0.18372339010238647, |
| "sampling/sampling_logp_difference/max": 1.694324016571045, |
| "sampling/sampling_logp_difference/mean": 0.019832005724310875, |
| "step": 4, |
| "step_time": 419.7083154738648 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16885.0, |
| "completions/max_terminated_length": 16885.0, |
| "completions/mean_length": 12227.375, |
| "completions/mean_terminated_length": 12227.375, |
| "completions/min_length": 9054.0, |
| "completions/min_terminated_length": 9054.0, |
| "entropy": 0.2566723832860589, |
| "epoch": 0.006002400960384154, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3329413831233978, |
| "learning_rate": 9.84e-07, |
| "loss": 0.021, |
| "num_tokens": 1853891.0, |
| "reward": 1.8860834836959839, |
| "reward_std": 2.3241159915924072, |
| "rewards/reward_func/mean": 1.8933619260787964, |
| "rewards/reward_func/std": 2.3200485706329346, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0072784423828125, |
| "rewards/soft_overlong_punishment_reward/std": 0.028263933956623077, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9909947514533997, |
| "sampling/importance_sampling_ratio/min": 0.17708024382591248, |
| "sampling/sampling_logp_difference/max": 1.7311522960662842, |
| "sampling/sampling_logp_difference/mean": 0.018662026152014732, |
| "step": 5, |
| "step_time": 244.05006601125933 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16596.0, |
| "completions/max_terminated_length": 16596.0, |
| "completions/mean_length": 10645.0625, |
| "completions/mean_terminated_length": 10645.0625, |
| "completions/min_length": 7554.0, |
| "completions/min_terminated_length": 7554.0, |
| "entropy": 0.27365364972501993, |
| "epoch": 0.007202881152460984, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40234512090682983, |
| "learning_rate": 9.8e-07, |
| "loss": 0.0528, |
| "num_tokens": 2223725.0, |
| "reward": 1.9188847541809082, |
| "reward_std": 2.996480703353882, |
| "rewards/reward_func/mean": 1.9205021858215332, |
| "rewards/reward_func/std": 2.995396375656128, |
| "rewards/soft_overlong_punishment_reward/mean": -0.001617431640625, |
| "rewards/soft_overlong_punishment_reward/std": 0.009149574674665928, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9904695749282837, |
| "sampling/importance_sampling_ratio/min": 0.10882613807916641, |
| "sampling/sampling_logp_difference/max": 2.218003749847412, |
| "sampling/sampling_logp_difference/mean": 0.019550971686840057, |
| "step": 6, |
| "step_time": 222.7570838360116 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16016.0, |
| "completions/max_terminated_length": 16016.0, |
| "completions/mean_length": 10611.75, |
| "completions/mean_terminated_length": 10611.75, |
| "completions/min_length": 3615.0, |
| "completions/min_terminated_length": 3615.0, |
| "entropy": 0.2584681212902069, |
| "epoch": 0.008403361344537815, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37323054671287537, |
| "learning_rate": 9.759999999999998e-07, |
| "loss": -0.034, |
| "num_tokens": 2600909.0, |
| "reward": 2.0009255409240723, |
| "reward_std": 2.1709108352661133, |
| "rewards/reward_func/mean": 2.0009255409240723, |
| "rewards/reward_func/std": 2.1709108352661133, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9910842776298523, |
| "sampling/importance_sampling_ratio/min": 0.12746387720108032, |
| "sampling/sampling_logp_difference/max": 2.059922218322754, |
| "sampling/sampling_logp_difference/mean": 0.018402449786663055, |
| "step": 7, |
| "step_time": 236.24530657555442 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15769.0, |
| "completions/max_terminated_length": 15769.0, |
| "completions/mean_length": 11196.375, |
| "completions/mean_terminated_length": 11196.375, |
| "completions/min_length": 7706.0, |
| "completions/min_terminated_length": 7706.0, |
| "entropy": 0.2538974490016699, |
| "epoch": 0.009603841536614645, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.23562167584896088, |
| "learning_rate": 9.72e-07, |
| "loss": 0.0366, |
| "num_tokens": 2999913.0, |
| "reward": 0.09837960451841354, |
| "reward_std": 0.23358193039894104, |
| "rewards/reward_func/mean": 0.09837960451841354, |
| "rewards/reward_func/std": 0.23358193039894104, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9912012815475464, |
| "sampling/importance_sampling_ratio/min": 0.16663774847984314, |
| "sampling/sampling_logp_difference/max": 1.953153371810913, |
| "sampling/sampling_logp_difference/mean": 0.018111014738678932, |
| "step": 8, |
| "step_time": 234.31651349924505 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15866.0, |
| "completions/max_terminated_length": 15866.0, |
| "completions/mean_length": 12010.46875, |
| "completions/mean_terminated_length": 12010.46875, |
| "completions/min_length": 8409.0, |
| "completions/min_terminated_length": 8409.0, |
| "entropy": 0.27975638769567013, |
| "epoch": 0.010804321728691477, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.376257061958313, |
| "learning_rate": 9.679999999999999e-07, |
| "loss": -0.0141, |
| "num_tokens": 3418312.0, |
| "reward": 3.2278504371643066, |
| "reward_std": 11.484737396240234, |
| "rewards/reward_func/mean": 3.2278504371643066, |
| "rewards/reward_func/std": 11.484737396240234, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.738037109375, |
| "sampling/importance_sampling_ratio/mean": 0.9900665879249573, |
| "sampling/importance_sampling_ratio/min": 0.0053693195804953575, |
| "sampling/sampling_logp_difference/max": 5.227054119110107, |
| "sampling/sampling_logp_difference/mean": 0.02012288197875023, |
| "step": 9, |
| "step_time": 239.36658373998944 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17639.0, |
| "completions/max_terminated_length": 17639.0, |
| "completions/mean_length": 11850.59375, |
| "completions/mean_terminated_length": 11850.59375, |
| "completions/min_length": 8366.0, |
| "completions/min_terminated_length": 8366.0, |
| "entropy": 0.27660834789276123, |
| "epoch": 0.012004801920768308, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3692915141582489, |
| "learning_rate": 9.64e-07, |
| "loss": 0.0535, |
| "num_tokens": 3824779.0, |
| "reward": 0.4762687683105469, |
| "reward_std": 0.7091255784034729, |
| "rewards/reward_func/mean": 0.4858436584472656, |
| "rewards/reward_func/std": 0.7002295255661011, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00957489013671875, |
| "rewards/soft_overlong_punishment_reward/std": 0.05416375771164894, |
| "sampling/importance_sampling_ratio/max": 2.2065083980560303, |
| "sampling/importance_sampling_ratio/mean": 0.9904176592826843, |
| "sampling/importance_sampling_ratio/min": 0.13561852276325226, |
| "sampling/sampling_logp_difference/max": 1.9979093074798584, |
| "sampling/sampling_logp_difference/mean": 0.019777128472924232, |
| "step": 10, |
| "step_time": 242.8805887659546 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17650.0, |
| "completions/max_terminated_length": 17650.0, |
| "completions/mean_length": 11499.65625, |
| "completions/mean_terminated_length": 11499.65625, |
| "completions/min_length": 7873.0, |
| "completions/min_terminated_length": 7873.0, |
| "entropy": 0.24676325172185898, |
| "epoch": 0.013205282112845138, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41307520866394043, |
| "learning_rate": 9.6e-07, |
| "loss": 0.0019, |
| "num_tokens": 4228592.0, |
| "reward": 1.2467007637023926, |
| "reward_std": 2.456725597381592, |
| "rewards/reward_func/mean": 1.256359577178955, |
| "rewards/reward_func/std": 2.451215982437134, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0096588134765625, |
| "rewards/soft_overlong_punishment_reward/std": 0.05463850125670433, |
| "sampling/importance_sampling_ratio/max": 2.828012466430664, |
| "sampling/importance_sampling_ratio/mean": 0.9914248585700989, |
| "sampling/importance_sampling_ratio/min": 0.36373454332351685, |
| "sampling/sampling_logp_difference/max": 1.039574146270752, |
| "sampling/sampling_logp_difference/mean": 0.01774602197110653, |
| "step": 11, |
| "step_time": 265.6139151197858 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14906.0, |
| "completions/max_terminated_length": 14906.0, |
| "completions/mean_length": 11677.15625, |
| "completions/mean_terminated_length": 11677.15625, |
| "completions/min_length": 7384.0, |
| "completions/min_terminated_length": 7384.0, |
| "entropy": 0.23969209752976894, |
| "epoch": 0.014405762304921969, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3561864495277405, |
| "learning_rate": 9.559999999999998e-07, |
| "loss": 0.0084, |
| "num_tokens": 4638621.0, |
| "reward": 0.6092424392700195, |
| "reward_std": 1.582262396812439, |
| "rewards/reward_func/mean": 0.6092424392700195, |
| "rewards/reward_func/std": 1.5822621583938599, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.7359044551849365, |
| "sampling/importance_sampling_ratio/mean": 0.9916994571685791, |
| "sampling/importance_sampling_ratio/min": 0.26353466510772705, |
| "sampling/sampling_logp_difference/max": 1.3335703611373901, |
| "sampling/sampling_logp_difference/mean": 0.017739253118634224, |
| "step": 12, |
| "step_time": 233.1278730876511 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17077.0, |
| "completions/max_terminated_length": 17077.0, |
| "completions/mean_length": 10545.90625, |
| "completions/mean_terminated_length": 10545.90625, |
| "completions/min_length": 7467.0, |
| "completions/min_terminated_length": 7467.0, |
| "entropy": 0.2735821781679988, |
| "epoch": 0.015606242496998799, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3848613202571869, |
| "learning_rate": 9.52e-07, |
| "loss": 0.0584, |
| "num_tokens": 5005954.0, |
| "reward": 2.2471165657043457, |
| "reward_std": 3.020482301712036, |
| "rewards/reward_func/mean": 2.253136157989502, |
| "rewards/reward_func/std": 3.0157971382141113, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00601959228515625, |
| "rewards/soft_overlong_punishment_reward/std": 0.030061684548854828, |
| "sampling/importance_sampling_ratio/max": 2.850409746170044, |
| "sampling/importance_sampling_ratio/mean": 0.9906243085861206, |
| "sampling/importance_sampling_ratio/min": 6.962251973163802e-06, |
| "sampling/sampling_logp_difference/max": 11.875007629394531, |
| "sampling/sampling_logp_difference/mean": 0.019261419773101807, |
| "step": 13, |
| "step_time": 243.16750305576716 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14631.0, |
| "completions/max_terminated_length": 14631.0, |
| "completions/mean_length": 10139.75, |
| "completions/mean_terminated_length": 10139.75, |
| "completions/min_length": 5047.0, |
| "completions/min_terminated_length": 5047.0, |
| "entropy": 0.24060232006013393, |
| "epoch": 0.01680672268907563, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.383498877286911, |
| "learning_rate": 9.479999999999999e-07, |
| "loss": 0.0478, |
| "num_tokens": 5374362.0, |
| "reward": 0.869640588760376, |
| "reward_std": 1.4982999563217163, |
| "rewards/reward_func/mean": 0.869640588760376, |
| "rewards/reward_func/std": 1.4982999563217163, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.358623743057251, |
| "sampling/importance_sampling_ratio/mean": 0.9915570616722107, |
| "sampling/importance_sampling_ratio/min": 0.07955317199230194, |
| "sampling/sampling_logp_difference/max": 2.53132963180542, |
| "sampling/sampling_logp_difference/mean": 0.017563849687576294, |
| "step": 14, |
| "step_time": 217.46682468871586 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17053.0, |
| "completions/max_terminated_length": 17053.0, |
| "completions/mean_length": 13305.40625, |
| "completions/mean_terminated_length": 13305.40625, |
| "completions/min_length": 10198.0, |
| "completions/min_terminated_length": 10198.0, |
| "entropy": 0.2278358107432723, |
| "epoch": 0.01800720288115246, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.29915085434913635, |
| "learning_rate": 9.439999999999999e-07, |
| "loss": 0.0318, |
| "num_tokens": 5840119.0, |
| "reward": 0.04119223356246948, |
| "reward_std": 0.2189895212650299, |
| "rewards/reward_func/mean": 0.04761618375778198, |
| "rewards/reward_func/std": 0.21552126109600067, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0064239501953125, |
| "rewards/soft_overlong_punishment_reward/std": 0.029588615521788597, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9919904470443726, |
| "sampling/importance_sampling_ratio/min": 0.06755267828702927, |
| "sampling/sampling_logp_difference/max": 2.694847583770752, |
| "sampling/sampling_logp_difference/mean": 0.01704443246126175, |
| "step": 15, |
| "step_time": 473.42477186385076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16258.0, |
| "completions/max_terminated_length": 16258.0, |
| "completions/mean_length": 10728.21875, |
| "completions/mean_terminated_length": 10728.21875, |
| "completions/min_length": 6511.0, |
| "completions/min_terminated_length": 6511.0, |
| "entropy": 0.2541895778849721, |
| "epoch": 0.01920768307322929, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.357921302318573, |
| "learning_rate": 9.399999999999999e-07, |
| "loss": -0.0349, |
| "num_tokens": 6216630.0, |
| "reward": 3.3305792808532715, |
| "reward_std": 7.61651611328125, |
| "rewards/reward_func/mean": 3.3305792808532715, |
| "rewards/reward_func/std": 7.61651611328125, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.522019624710083, |
| "sampling/importance_sampling_ratio/mean": 0.9909427762031555, |
| "sampling/importance_sampling_ratio/min": 0.14655600488185883, |
| "sampling/sampling_logp_difference/max": 1.9203475713729858, |
| "sampling/sampling_logp_difference/mean": 0.018465207889676094, |
| "step": 16, |
| "step_time": 229.7186487779254 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14954.0, |
| "completions/max_terminated_length": 14954.0, |
| "completions/mean_length": 10770.53125, |
| "completions/mean_terminated_length": 10770.53125, |
| "completions/min_length": 5937.0, |
| "completions/min_terminated_length": 5937.0, |
| "entropy": 0.2734638024121523, |
| "epoch": 0.02040816326530612, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.32195746898651123, |
| "learning_rate": 9.36e-07, |
| "loss": 0.0259, |
| "num_tokens": 6590607.0, |
| "reward": 0.6464040875434875, |
| "reward_std": 0.9011892676353455, |
| "rewards/reward_func/mean": 0.6464040875434875, |
| "rewards/reward_func/std": 0.9011892676353455, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1569271087646484, |
| "sampling/importance_sampling_ratio/mean": 0.9905099868774414, |
| "sampling/importance_sampling_ratio/min": 0.02047974243760109, |
| "sampling/sampling_logp_difference/max": 3.8883190155029297, |
| "sampling/sampling_logp_difference/mean": 0.019440319389104843, |
| "step": 17, |
| "step_time": 433.71242073585745 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16352.0, |
| "completions/max_terminated_length": 16352.0, |
| "completions/mean_length": 9984.46875, |
| "completions/mean_terminated_length": 9984.46875, |
| "completions/min_length": 4147.0, |
| "completions/min_terminated_length": 4147.0, |
| "entropy": 0.2726652082055807, |
| "epoch": 0.021608643457382955, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3855082392692566, |
| "learning_rate": 9.32e-07, |
| "loss": 0.0234, |
| "num_tokens": 6944518.0, |
| "reward": 1.9245760440826416, |
| "reward_std": 2.4882936477661133, |
| "rewards/reward_func/mean": 1.9245760440826416, |
| "rewards/reward_func/std": 2.4882936477661133, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0238037109375, |
| "sampling/importance_sampling_ratio/mean": 0.9904158115386963, |
| "sampling/importance_sampling_ratio/min": 0.19708161056041718, |
| "sampling/sampling_logp_difference/max": 1.6241374015808105, |
| "sampling/sampling_logp_difference/mean": 0.01931804046034813, |
| "step": 18, |
| "step_time": 261.609183562221 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19890.0, |
| "completions/max_terminated_length": 19890.0, |
| "completions/mean_length": 12736.40625, |
| "completions/mean_terminated_length": 12736.40625, |
| "completions/min_length": 8136.0, |
| "completions/min_terminated_length": 8136.0, |
| "entropy": 0.22564240265637636, |
| "epoch": 0.022809123649459785, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3456736207008362, |
| "learning_rate": 9.28e-07, |
| "loss": 0.0822, |
| "num_tokens": 7391411.0, |
| "reward": 55.75565719604492, |
| "reward_std": 127.06681823730469, |
| "rewards/reward_func/mean": 55.820682525634766, |
| "rewards/reward_func/std": 127.06536102294922, |
| "rewards/soft_overlong_punishment_reward/mean": -0.06502532958984375, |
| "rewards/soft_overlong_punishment_reward/std": 0.21553081274032593, |
| "sampling/importance_sampling_ratio/max": 2.4781954288482666, |
| "sampling/importance_sampling_ratio/mean": 0.9922029376029968, |
| "sampling/importance_sampling_ratio/min": 0.18941795825958252, |
| "sampling/sampling_logp_difference/max": 1.6637992858886719, |
| "sampling/sampling_logp_difference/mean": 0.016782237216830254, |
| "step": 19, |
| "step_time": 274.4253786377376 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 18974.0, |
| "completions/max_terminated_length": 18974.0, |
| "completions/mean_length": 11451.9375, |
| "completions/mean_terminated_length": 11451.9375, |
| "completions/min_length": 8223.0, |
| "completions/min_terminated_length": 8223.0, |
| "entropy": 0.24693564232438803, |
| "epoch": 0.024009603841536616, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37484318017959595, |
| "learning_rate": 9.24e-07, |
| "loss": 0.0537, |
| "num_tokens": 7801249.0, |
| "reward": 1.6996723413467407, |
| "reward_std": 2.6643435955047607, |
| "rewards/reward_func/mean": 1.7194324731826782, |
| "rewards/reward_func/std": 2.648789882659912, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0197601318359375, |
| "rewards/soft_overlong_punishment_reward/std": 0.11178018897771835, |
| "sampling/importance_sampling_ratio/max": 2.0502030849456787, |
| "sampling/importance_sampling_ratio/mean": 0.991265594959259, |
| "sampling/importance_sampling_ratio/min": 0.03500676155090332, |
| "sampling/sampling_logp_difference/max": 3.3522140979766846, |
| "sampling/sampling_logp_difference/mean": 0.017859291285276413, |
| "step": 20, |
| "step_time": 254.13871470477898 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14921.0, |
| "completions/max_terminated_length": 14921.0, |
| "completions/mean_length": 11261.625, |
| "completions/mean_terminated_length": 11261.625, |
| "completions/min_length": 8921.0, |
| "completions/min_terminated_length": 8921.0, |
| "entropy": 0.21324953343719244, |
| "epoch": 0.025210084033613446, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37738341093063354, |
| "learning_rate": 9.2e-07, |
| "loss": 0.0006, |
| "num_tokens": 8209717.0, |
| "reward": 5.487376689910889, |
| "reward_std": 5.091497421264648, |
| "rewards/reward_func/mean": 5.487376689910889, |
| "rewards/reward_func/std": 5.091497421264648, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.863360643386841, |
| "sampling/importance_sampling_ratio/mean": 0.9924338459968567, |
| "sampling/importance_sampling_ratio/min": 0.2230137288570404, |
| "sampling/sampling_logp_difference/max": 1.5005220174789429, |
| "sampling/sampling_logp_difference/mean": 0.01615520380437374, |
| "step": 21, |
| "step_time": 226.67027153016534 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15897.0, |
| "completions/max_terminated_length": 15897.0, |
| "completions/mean_length": 9916.03125, |
| "completions/mean_terminated_length": 9916.03125, |
| "completions/min_length": 6559.0, |
| "completions/min_terminated_length": 6559.0, |
| "entropy": 0.2823543632403016, |
| "epoch": 0.026410564225690276, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41728851199150085, |
| "learning_rate": 9.16e-07, |
| "loss": 0.0357, |
| "num_tokens": 8558022.0, |
| "reward": 1.0093857049942017, |
| "reward_std": 1.6929666996002197, |
| "rewards/reward_func/mean": 1.0093857049942017, |
| "rewards/reward_func/std": 1.6929665803909302, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9902706146240234, |
| "sampling/importance_sampling_ratio/min": 0.29672637581825256, |
| "sampling/sampling_logp_difference/max": 1.2743182182312012, |
| "sampling/sampling_logp_difference/mean": 0.020027777180075645, |
| "step": 22, |
| "step_time": 210.78112597286236 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15085.0, |
| "completions/max_terminated_length": 15085.0, |
| "completions/mean_length": 9004.78125, |
| "completions/mean_terminated_length": 9004.78125, |
| "completions/min_length": 5019.0, |
| "completions/min_terminated_length": 5019.0, |
| "entropy": 0.26619420014321804, |
| "epoch": 0.027611044417767107, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4362182021141052, |
| "learning_rate": 9.12e-07, |
| "loss": -0.0028, |
| "num_tokens": 8867615.0, |
| "reward": 3.192936420440674, |
| "reward_std": 3.843594551086426, |
| "rewards/reward_func/mean": 3.192936420440674, |
| "rewards/reward_func/std": 3.8435938358306885, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.684079170227051, |
| "sampling/importance_sampling_ratio/mean": 0.9908168315887451, |
| "sampling/importance_sampling_ratio/min": 0.24921053647994995, |
| "sampling/sampling_logp_difference/max": 1.3894572257995605, |
| "sampling/sampling_logp_difference/mean": 0.019214916974306107, |
| "step": 23, |
| "step_time": 200.50691007694695 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15050.0, |
| "completions/max_terminated_length": 15050.0, |
| "completions/mean_length": 10668.5625, |
| "completions/mean_terminated_length": 10668.5625, |
| "completions/min_length": 6005.0, |
| "completions/min_terminated_length": 6005.0, |
| "entropy": 0.2756733503192663, |
| "epoch": 0.028811524609843937, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37513336539268494, |
| "learning_rate": 9.08e-07, |
| "loss": 0.0388, |
| "num_tokens": 9237721.0, |
| "reward": 0.4544358551502228, |
| "reward_std": 0.5964015126228333, |
| "rewards/reward_func/mean": 0.4544358551502228, |
| "rewards/reward_func/std": 0.5964015126228333, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.185565233230591, |
| "sampling/importance_sampling_ratio/mean": 0.9904376268386841, |
| "sampling/importance_sampling_ratio/min": 0.3395426869392395, |
| "sampling/sampling_logp_difference/max": 1.080155611038208, |
| "sampling/sampling_logp_difference/mean": 0.019677145406603813, |
| "step": 24, |
| "step_time": 225.78321035497356 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14237.0, |
| "completions/max_terminated_length": 14237.0, |
| "completions/mean_length": 10287.375, |
| "completions/mean_terminated_length": 10287.375, |
| "completions/min_length": 6909.0, |
| "completions/min_terminated_length": 6909.0, |
| "entropy": 0.2614977052435279, |
| "epoch": 0.030012004801920768, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3477057218551636, |
| "learning_rate": 9.039999999999999e-07, |
| "loss": 0.0355, |
| "num_tokens": 9600461.0, |
| "reward": 1.3245511054992676, |
| "reward_std": 2.4857571125030518, |
| "rewards/reward_func/mean": 1.3245511054992676, |
| "rewards/reward_func/std": 2.4857571125030518, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.117020606994629, |
| "sampling/importance_sampling_ratio/mean": 0.9908631443977356, |
| "sampling/importance_sampling_ratio/min": 0.12998507916927338, |
| "sampling/sampling_logp_difference/max": 2.0403356552124023, |
| "sampling/sampling_logp_difference/mean": 0.018762774765491486, |
| "step": 25, |
| "step_time": 207.05053522088565 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14980.0, |
| "completions/max_terminated_length": 14980.0, |
| "completions/mean_length": 9842.53125, |
| "completions/mean_terminated_length": 9842.53125, |
| "completions/min_length": 6382.0, |
| "completions/min_terminated_length": 6382.0, |
| "entropy": 0.2648027054965496, |
| "epoch": 0.031212484993997598, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3818010091781616, |
| "learning_rate": 9e-07, |
| "loss": 0.0235, |
| "num_tokens": 9945774.0, |
| "reward": 1.9594688415527344, |
| "reward_std": 1.9573324918746948, |
| "rewards/reward_func/mean": 1.9594688415527344, |
| "rewards/reward_func/std": 1.9573323726654053, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.115938425064087, |
| "sampling/importance_sampling_ratio/mean": 0.9907746315002441, |
| "sampling/importance_sampling_ratio/min": 0.16936549544334412, |
| "sampling/sampling_logp_difference/max": 1.7756961584091187, |
| "sampling/sampling_logp_difference/mean": 0.018964044749736786, |
| "step": 26, |
| "step_time": 210.5417184109101 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14752.0, |
| "completions/max_terminated_length": 14752.0, |
| "completions/mean_length": 10530.3125, |
| "completions/mean_terminated_length": 10530.3125, |
| "completions/min_length": 6272.0, |
| "completions/min_terminated_length": 6272.0, |
| "entropy": 0.24934661202132702, |
| "epoch": 0.03241296518607443, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40029430389404297, |
| "learning_rate": 8.96e-07, |
| "loss": -0.0036, |
| "num_tokens": 10312320.0, |
| "reward": 3.0405454635620117, |
| "reward_std": 3.8312883377075195, |
| "rewards/reward_func/mean": 3.0405454635620117, |
| "rewards/reward_func/std": 3.8312880992889404, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9914329648017883, |
| "sampling/importance_sampling_ratio/min": 0.09976457804441452, |
| "sampling/sampling_logp_difference/max": 2.3049421310424805, |
| "sampling/sampling_logp_difference/mean": 0.017998939380049706, |
| "step": 27, |
| "step_time": 431.60967801115476 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17459.0, |
| "completions/max_terminated_length": 17459.0, |
| "completions/mean_length": 11231.8125, |
| "completions/mean_terminated_length": 11231.8125, |
| "completions/min_length": 5633.0, |
| "completions/min_terminated_length": 5633.0, |
| "entropy": 0.25841675139963627, |
| "epoch": 0.03361344537815126, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3741910755634308, |
| "learning_rate": 8.92e-07, |
| "loss": 0.0071, |
| "num_tokens": 10709450.0, |
| "reward": 1.4909887313842773, |
| "reward_std": 2.263838291168213, |
| "rewards/reward_func/mean": 1.499190330505371, |
| "rewards/reward_func/std": 2.2577483654022217, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00820159912109375, |
| "rewards/soft_overlong_punishment_reward/std": 0.04639524966478348, |
| "sampling/importance_sampling_ratio/max": 2.4496078491210938, |
| "sampling/importance_sampling_ratio/mean": 0.99090176820755, |
| "sampling/importance_sampling_ratio/min": 0.13220876455307007, |
| "sampling/sampling_logp_difference/max": 2.0233731269836426, |
| "sampling/sampling_logp_difference/mean": 0.01868780329823494, |
| "step": 28, |
| "step_time": 426.57934639882296 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16356.0, |
| "completions/max_terminated_length": 16356.0, |
| "completions/mean_length": 9573.6875, |
| "completions/mean_terminated_length": 9573.6875, |
| "completions/min_length": 2818.0, |
| "completions/min_terminated_length": 2818.0, |
| "entropy": 0.2775294054299593, |
| "epoch": 0.03481392557022809, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3293408453464508, |
| "learning_rate": 8.88e-07, |
| "loss": 0.0192, |
| "num_tokens": 11044352.0, |
| "reward": 1.3649942874908447, |
| "reward_std": 1.9040188789367676, |
| "rewards/reward_func/mean": 1.3649942874908447, |
| "rewards/reward_func/std": 1.9040188789367676, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.990324854850769, |
| "sampling/importance_sampling_ratio/min": 0.37744393944740295, |
| "sampling/sampling_logp_difference/max": 1.3116214275360107, |
| "sampling/sampling_logp_difference/mean": 0.019651668146252632, |
| "step": 29, |
| "step_time": 238.1490348140942 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14524.0, |
| "completions/max_terminated_length": 14524.0, |
| "completions/mean_length": 11151.8125, |
| "completions/mean_terminated_length": 11151.8125, |
| "completions/min_length": 7531.0, |
| "completions/min_terminated_length": 7531.0, |
| "entropy": 0.24554751068353653, |
| "epoch": 0.03601440576230492, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36097458004951477, |
| "learning_rate": 8.839999999999999e-07, |
| "loss": -0.0026, |
| "num_tokens": 11442354.0, |
| "reward": 2.1457722187042236, |
| "reward_std": 2.174523115158081, |
| "rewards/reward_func/mean": 2.1457722187042236, |
| "rewards/reward_func/std": 2.17452335357666, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9914038777351379, |
| "sampling/importance_sampling_ratio/min": 0.1909109503030777, |
| "sampling/sampling_logp_difference/max": 1.6559481620788574, |
| "sampling/sampling_logp_difference/mean": 0.0179743692278862, |
| "step": 30, |
| "step_time": 251.35512228007428 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13663.0, |
| "completions/max_terminated_length": 13663.0, |
| "completions/mean_length": 8797.78125, |
| "completions/mean_terminated_length": 8797.78125, |
| "completions/min_length": 3457.0, |
| "completions/min_terminated_length": 3457.0, |
| "entropy": 0.26490489952266216, |
| "epoch": 0.03721488595438175, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.45504486560821533, |
| "learning_rate": 8.799999999999999e-07, |
| "loss": 0.0903, |
| "num_tokens": 11752779.0, |
| "reward": 2.032346725463867, |
| "reward_std": 2.6423678398132324, |
| "rewards/reward_func/mean": 2.032346725463867, |
| "rewards/reward_func/std": 2.6423676013946533, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9907644987106323, |
| "sampling/importance_sampling_ratio/min": 0.12091671675443649, |
| "sampling/sampling_logp_difference/max": 2.1126532554626465, |
| "sampling/sampling_logp_difference/mean": 0.01906166598200798, |
| "step": 31, |
| "step_time": 210.08964343706612 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 17167.0, |
| "completions/mean_length": 10623.3125, |
| "completions/mean_terminated_length": 10305.3544921875, |
| "completions/min_length": 4879.0, |
| "completions/min_terminated_length": 4879.0, |
| "entropy": 0.2628009468317032, |
| "epoch": 0.03841536614645858, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.2900553047657013, |
| "learning_rate": 8.76e-07, |
| "loss": -0.0423, |
| "num_tokens": 12126181.0, |
| "reward": 1.0021374225616455, |
| "reward_std": 1.9135971069335938, |
| "rewards/reward_func/mean": 1.0393612384796143, |
| "rewards/reward_func/std": 1.8841371536254883, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03722381591796875, |
| "rewards/soft_overlong_punishment_reward/std": 0.17890377342700958, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9907552599906921, |
| "sampling/importance_sampling_ratio/min": 0.2805570363998413, |
| "sampling/sampling_logp_difference/max": 1.2783308029174805, |
| "sampling/sampling_logp_difference/mean": 0.018829116597771645, |
| "step": 32, |
| "step_time": 280.71025710599497 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14347.0, |
| "completions/max_terminated_length": 14347.0, |
| "completions/mean_length": 10457.4375, |
| "completions/mean_terminated_length": 10457.4375, |
| "completions/min_length": 7484.0, |
| "completions/min_terminated_length": 7484.0, |
| "entropy": 0.28439216688275337, |
| "epoch": 0.03961584633853541, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3339356482028961, |
| "learning_rate": 8.72e-07, |
| "loss": 0.031, |
| "num_tokens": 12492219.0, |
| "reward": 0.7305504083633423, |
| "reward_std": 0.7552641034126282, |
| "rewards/reward_func/mean": 0.7305504083633423, |
| "rewards/reward_func/std": 0.7552641034126282, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9899792671203613, |
| "sampling/importance_sampling_ratio/min": 0.15883956849575043, |
| "sampling/sampling_logp_difference/max": 1.8398605585098267, |
| "sampling/sampling_logp_difference/mean": 0.020156892016530037, |
| "step": 33, |
| "step_time": 222.89753303222824 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14362.0, |
| "completions/max_terminated_length": 14362.0, |
| "completions/mean_length": 10071.5, |
| "completions/mean_terminated_length": 10071.5, |
| "completions/min_length": 6369.0, |
| "completions/min_terminated_length": 6369.0, |
| "entropy": 0.24872859101742506, |
| "epoch": 0.04081632653061224, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4110364019870758, |
| "learning_rate": 8.68e-07, |
| "loss": -0.0135, |
| "num_tokens": 12855851.0, |
| "reward": 2.7421875, |
| "reward_std": 3.5190269947052, |
| "rewards/reward_func/mean": 2.7421875, |
| "rewards/reward_func/std": 3.5190269947052, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9915586113929749, |
| "sampling/importance_sampling_ratio/min": 0.27868548035621643, |
| "sampling/sampling_logp_difference/max": 1.2776713371276855, |
| "sampling/sampling_logp_difference/mean": 0.01771131530404091, |
| "step": 34, |
| "step_time": 224.99640086409636 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13259.0, |
| "completions/max_terminated_length": 13259.0, |
| "completions/mean_length": 8307.40625, |
| "completions/mean_terminated_length": 8307.40625, |
| "completions/min_length": 3245.0, |
| "completions/min_terminated_length": 3245.0, |
| "entropy": 0.2815095093101263, |
| "epoch": 0.04201680672268908, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4219931364059448, |
| "learning_rate": 8.639999999999999e-07, |
| "loss": 0.02, |
| "num_tokens": 13148672.0, |
| "reward": 1.6012442111968994, |
| "reward_std": 2.0349299907684326, |
| "rewards/reward_func/mean": 1.6012442111968994, |
| "rewards/reward_func/std": 2.0349299907684326, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3349249362945557, |
| "sampling/importance_sampling_ratio/mean": 0.990214467048645, |
| "sampling/importance_sampling_ratio/min": 0.10321995615959167, |
| "sampling/sampling_logp_difference/max": 2.270893096923828, |
| "sampling/sampling_logp_difference/mean": 0.019673598930239677, |
| "step": 35, |
| "step_time": 197.53019676276017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13594.0, |
| "completions/max_terminated_length": 13594.0, |
| "completions/mean_length": 10127.0625, |
| "completions/mean_terminated_length": 10127.0625, |
| "completions/min_length": 6623.0, |
| "completions/min_terminated_length": 6623.0, |
| "entropy": 0.2668821206316352, |
| "epoch": 0.04321728691476591, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37350013852119446, |
| "learning_rate": 8.599999999999999e-07, |
| "loss": 0.0004, |
| "num_tokens": 13502514.0, |
| "reward": 2.3392128944396973, |
| "reward_std": 3.019270896911621, |
| "rewards/reward_func/mean": 2.3392128944396973, |
| "rewards/reward_func/std": 3.019270896911621, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.516148328781128, |
| "sampling/importance_sampling_ratio/mean": 0.9906859397888184, |
| "sampling/importance_sampling_ratio/min": 0.2636016309261322, |
| "sampling/sampling_logp_difference/max": 1.3333163261413574, |
| "sampling/sampling_logp_difference/mean": 0.01926989108324051, |
| "step": 36, |
| "step_time": 219.1794932379853 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15015.0, |
| "completions/max_terminated_length": 15015.0, |
| "completions/mean_length": 10019.1875, |
| "completions/mean_terminated_length": 10019.1875, |
| "completions/min_length": 6810.0, |
| "completions/min_terminated_length": 6810.0, |
| "entropy": 0.24516229517757893, |
| "epoch": 0.04441776710684274, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37556055188179016, |
| "learning_rate": 8.559999999999999e-07, |
| "loss": -0.0232, |
| "num_tokens": 13860184.0, |
| "reward": 5.546761512756348, |
| "reward_std": 4.9547014236450195, |
| "rewards/reward_func/mean": 5.546761512756348, |
| "rewards/reward_func/std": 4.954700946807861, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9915067553520203, |
| "sampling/importance_sampling_ratio/min": 0.3167104423046112, |
| "sampling/sampling_logp_difference/max": 1.2001211643218994, |
| "sampling/sampling_logp_difference/mean": 0.017740583047270775, |
| "step": 37, |
| "step_time": 429.79983439377975 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15132.0, |
| "completions/max_terminated_length": 15132.0, |
| "completions/mean_length": 10126.53125, |
| "completions/mean_terminated_length": 10126.53125, |
| "completions/min_length": 3623.0, |
| "completions/min_terminated_length": 3623.0, |
| "entropy": 0.2536043291911483, |
| "epoch": 0.04561824729891957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4053056538105011, |
| "learning_rate": 8.52e-07, |
| "loss": 0.0016, |
| "num_tokens": 14219801.0, |
| "reward": 1.1494144201278687, |
| "reward_std": 1.515267252922058, |
| "rewards/reward_func/mean": 1.1494144201278687, |
| "rewards/reward_func/std": 1.515267252922058, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.504192590713501, |
| "sampling/importance_sampling_ratio/mean": 0.9913269281387329, |
| "sampling/importance_sampling_ratio/min": 0.11853170394897461, |
| "sampling/sampling_logp_difference/max": 2.1325747966766357, |
| "sampling/sampling_logp_difference/mean": 0.018079929053783417, |
| "step": 38, |
| "step_time": 231.09112938609906 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13953.0, |
| "completions/max_terminated_length": 13953.0, |
| "completions/mean_length": 9400.75, |
| "completions/mean_terminated_length": 9400.75, |
| "completions/min_length": 3400.0, |
| "completions/min_terminated_length": 3400.0, |
| "entropy": 0.2656275276094675, |
| "epoch": 0.0468187274909964, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.35668453574180603, |
| "learning_rate": 8.48e-07, |
| "loss": 0.0444, |
| "num_tokens": 14552409.0, |
| "reward": 1.375227689743042, |
| "reward_std": 2.0771355628967285, |
| "rewards/reward_func/mean": 1.375227689743042, |
| "rewards/reward_func/std": 2.0771355628967285, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8366079330444336, |
| "sampling/importance_sampling_ratio/mean": 0.9909170269966125, |
| "sampling/importance_sampling_ratio/min": 0.021294279024004936, |
| "sampling/sampling_logp_difference/max": 3.8493168354034424, |
| "sampling/sampling_logp_difference/mean": 0.018923122435808182, |
| "step": 39, |
| "step_time": 205.99593690189067 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14127.0, |
| "completions/max_terminated_length": 14127.0, |
| "completions/mean_length": 9623.75, |
| "completions/mean_terminated_length": 9623.75, |
| "completions/min_length": 5045.0, |
| "completions/min_terminated_length": 5045.0, |
| "entropy": 0.22837793547660112, |
| "epoch": 0.04801920768307323, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.344384104013443, |
| "learning_rate": 8.439999999999999e-07, |
| "loss": 0.0052, |
| "num_tokens": 14898929.0, |
| "reward": 0.5646069049835205, |
| "reward_std": 1.1903877258300781, |
| "rewards/reward_func/mean": 0.5646069049835205, |
| "rewards/reward_func/std": 1.1903877258300781, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.5692195892333984, |
| "sampling/importance_sampling_ratio/mean": 0.9922388195991516, |
| "sampling/importance_sampling_ratio/min": 0.14704354107379913, |
| "sampling/sampling_logp_difference/max": 1.9170265197753906, |
| "sampling/sampling_logp_difference/mean": 0.01625021919608116, |
| "step": 40, |
| "step_time": 228.8796401922591 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16700.0, |
| "completions/max_terminated_length": 16700.0, |
| "completions/mean_length": 11196.0, |
| "completions/mean_terminated_length": 11196.0, |
| "completions/min_length": 7400.0, |
| "completions/min_terminated_length": 7400.0, |
| "entropy": 0.23109178990125656, |
| "epoch": 0.04921968787515006, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.35686036944389343, |
| "learning_rate": 8.399999999999999e-07, |
| "loss": -0.025, |
| "num_tokens": 15295537.0, |
| "reward": 1.7151178121566772, |
| "reward_std": 2.868417501449585, |
| "rewards/reward_func/mean": 1.7175287008285522, |
| "rewards/reward_func/std": 2.868037700653076, |
| "rewards/soft_overlong_punishment_reward/mean": -0.002410888671875, |
| "rewards/soft_overlong_punishment_reward/std": 0.013638045638799667, |
| "sampling/importance_sampling_ratio/max": 2.9357402324676514, |
| "sampling/importance_sampling_ratio/mean": 0.9918723106384277, |
| "sampling/importance_sampling_ratio/min": 0.3307071328163147, |
| "sampling/sampling_logp_difference/max": 1.1065220832824707, |
| "sampling/sampling_logp_difference/mean": 0.016960658133029938, |
| "step": 41, |
| "step_time": 247.38727062719408 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11833.0, |
| "completions/max_terminated_length": 11833.0, |
| "completions/mean_length": 9130.28125, |
| "completions/mean_terminated_length": 9130.28125, |
| "completions/min_length": 6003.0, |
| "completions/min_terminated_length": 6003.0, |
| "entropy": 0.27010741643607616, |
| "epoch": 0.05042016806722689, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.36788010597229004, |
| "learning_rate": 8.359999999999999e-07, |
| "loss": 0.0177, |
| "num_tokens": 15624346.0, |
| "reward": 0.4658914804458618, |
| "reward_std": 0.9191152453422546, |
| "rewards/reward_func/mean": 0.4658914804458618, |
| "rewards/reward_func/std": 0.9191153049468994, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9905910491943359, |
| "sampling/importance_sampling_ratio/min": 0.02883448265492916, |
| "sampling/sampling_logp_difference/max": 3.5461833477020264, |
| "sampling/sampling_logp_difference/mean": 0.01935204118490219, |
| "step": 42, |
| "step_time": 186.53937864280306 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15715.0, |
| "completions/max_terminated_length": 15715.0, |
| "completions/mean_length": 11385.625, |
| "completions/mean_terminated_length": 11385.625, |
| "completions/min_length": 5736.0, |
| "completions/min_terminated_length": 5736.0, |
| "entropy": 0.2562467288225889, |
| "epoch": 0.05162064825930372, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.31052911281585693, |
| "learning_rate": 8.319999999999999e-07, |
| "loss": -0.0051, |
| "num_tokens": 16022038.0, |
| "reward": 1.6798204183578491, |
| "reward_std": 2.83245587348938, |
| "rewards/reward_func/mean": 1.6798204183578491, |
| "rewards/reward_func/std": 2.83245587348938, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.5458321571350098, |
| "sampling/importance_sampling_ratio/mean": 0.9909483790397644, |
| "sampling/importance_sampling_ratio/min": 0.2335936576128006, |
| "sampling/sampling_logp_difference/max": 1.454172134399414, |
| "sampling/sampling_logp_difference/mean": 0.01847730576992035, |
| "step": 43, |
| "step_time": 446.1300635159714 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17778.0, |
| "completions/max_terminated_length": 17778.0, |
| "completions/mean_length": 9443.03125, |
| "completions/mean_terminated_length": 9443.03125, |
| "completions/min_length": 4794.0, |
| "completions/min_terminated_length": 4794.0, |
| "entropy": 0.25649653095752, |
| "epoch": 0.05282112845138055, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4464053809642792, |
| "learning_rate": 8.28e-07, |
| "loss": 0.0453, |
| "num_tokens": 16358935.0, |
| "reward": 1.2445961236953735, |
| "reward_std": 2.735797643661499, |
| "rewards/reward_func/mean": 1.255231499671936, |
| "rewards/reward_func/std": 2.730092763900757, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0106353759765625, |
| "rewards/soft_overlong_punishment_reward/std": 0.060162775218486786, |
| "sampling/importance_sampling_ratio/max": 2.382478713989258, |
| "sampling/importance_sampling_ratio/mean": 0.9911012649536133, |
| "sampling/importance_sampling_ratio/min": 0.2831081748008728, |
| "sampling/sampling_logp_difference/max": 1.2619261741638184, |
| "sampling/sampling_logp_difference/mean": 0.018203921616077423, |
| "step": 44, |
| "step_time": 238.45520115306135 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 18164.0, |
| "completions/max_terminated_length": 18164.0, |
| "completions/mean_length": 11039.0625, |
| "completions/mean_terminated_length": 11039.0625, |
| "completions/min_length": 6961.0, |
| "completions/min_terminated_length": 6961.0, |
| "entropy": 0.26013822108507156, |
| "epoch": 0.05402160864345738, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.7753751873970032, |
| "learning_rate": 8.24e-07, |
| "loss": 0.0627, |
| "num_tokens": 16746577.0, |
| "reward": 0.7348491549491882, |
| "reward_std": 1.1186299324035645, |
| "rewards/reward_func/mean": 0.756905734539032, |
| "rewards/reward_func/std": 1.0995012521743774, |
| "rewards/soft_overlong_punishment_reward/mean": -0.02205657958984375, |
| "rewards/soft_overlong_punishment_reward/std": 0.08923598378896713, |
| "sampling/importance_sampling_ratio/max": 2.8484528064727783, |
| "sampling/importance_sampling_ratio/mean": 0.9910010695457458, |
| "sampling/importance_sampling_ratio/min": 0.23139524459838867, |
| "sampling/sampling_logp_difference/max": 1.4636280536651611, |
| "sampling/sampling_logp_difference/mean": 0.01876596175134182, |
| "step": 45, |
| "step_time": 266.24963577114977 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16171.0, |
| "completions/max_terminated_length": 16171.0, |
| "completions/mean_length": 10262.96875, |
| "completions/mean_terminated_length": 10262.96875, |
| "completions/min_length": 5913.0, |
| "completions/min_terminated_length": 5913.0, |
| "entropy": 0.25811031460762024, |
| "epoch": 0.055222088835534214, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.378570556640625, |
| "learning_rate": 8.199999999999999e-07, |
| "loss": 0.0386, |
| "num_tokens": 17105832.0, |
| "reward": 2.625810146331787, |
| "reward_std": 3.4443087577819824, |
| "rewards/reward_func/mean": 2.625810146331787, |
| "rewards/reward_func/std": 3.4443087577819824, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0112507343292236, |
| "sampling/importance_sampling_ratio/mean": 0.9910373091697693, |
| "sampling/importance_sampling_ratio/min": 0.15773901343345642, |
| "sampling/sampling_logp_difference/max": 1.846813440322876, |
| "sampling/sampling_logp_difference/mean": 0.01831601746380329, |
| "step": 46, |
| "step_time": 237.72815251280554 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15380.0, |
| "completions/max_terminated_length": 15380.0, |
| "completions/mean_length": 8343.25, |
| "completions/mean_terminated_length": 8343.25, |
| "completions/min_length": 4085.0, |
| "completions/min_terminated_length": 4085.0, |
| "entropy": 0.264944227412343, |
| "epoch": 0.056422569027611044, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4716252088546753, |
| "learning_rate": 8.159999999999999e-07, |
| "loss": 0.0628, |
| "num_tokens": 17399992.0, |
| "reward": 2.4003467559814453, |
| "reward_std": 3.171684741973877, |
| "rewards/reward_func/mean": 2.4003467559814453, |
| "rewards/reward_func/std": 3.171684503555298, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.402794599533081, |
| "sampling/importance_sampling_ratio/mean": 0.9908208250999451, |
| "sampling/importance_sampling_ratio/min": 0.3649902939796448, |
| "sampling/sampling_logp_difference/max": 1.0078845024108887, |
| "sampling/sampling_logp_difference/mean": 0.018845025449991226, |
| "step": 47, |
| "step_time": 209.52517059491947 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14507.0, |
| "completions/max_terminated_length": 14507.0, |
| "completions/mean_length": 10999.65625, |
| "completions/mean_terminated_length": 10999.65625, |
| "completions/min_length": 7392.0, |
| "completions/min_terminated_length": 7392.0, |
| "entropy": 0.2484130971133709, |
| "epoch": 0.057623049219687875, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3668311834335327, |
| "learning_rate": 8.12e-07, |
| "loss": -0.0103, |
| "num_tokens": 17789653.0, |
| "reward": 0.8897507786750793, |
| "reward_std": 0.6408941149711609, |
| "rewards/reward_func/mean": 0.8897507786750793, |
| "rewards/reward_func/std": 0.6408940553665161, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.341449022293091, |
| "sampling/importance_sampling_ratio/mean": 0.9913567304611206, |
| "sampling/importance_sampling_ratio/min": 0.022933995351195335, |
| "sampling/sampling_logp_difference/max": 3.775135040283203, |
| "sampling/sampling_logp_difference/mean": 0.017700741067528725, |
| "step": 48, |
| "step_time": 227.09850537101738 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16482.0, |
| "completions/max_terminated_length": 16482.0, |
| "completions/mean_length": 11000.375, |
| "completions/mean_terminated_length": 11000.375, |
| "completions/min_length": 6194.0, |
| "completions/min_terminated_length": 6194.0, |
| "entropy": 0.2574823070317507, |
| "epoch": 0.058823529411764705, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37714579701423645, |
| "learning_rate": 8.08e-07, |
| "loss": 0.039, |
| "num_tokens": 18186617.0, |
| "reward": 1.1769545078277588, |
| "reward_std": 0.6496437788009644, |
| "rewards/reward_func/mean": 1.1777021884918213, |
| "rewards/reward_func/std": 0.6509500741958618, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0007476806640625, |
| "rewards/soft_overlong_punishment_reward/std": 0.004229520447552204, |
| "sampling/importance_sampling_ratio/max": 2.0076770782470703, |
| "sampling/importance_sampling_ratio/mean": 0.9911171197891235, |
| "sampling/importance_sampling_ratio/min": 0.015267138369381428, |
| "sampling/sampling_logp_difference/max": 4.1820526123046875, |
| "sampling/sampling_logp_difference/mean": 0.017983591184020042, |
| "step": 49, |
| "step_time": 450.96922948525753 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12987.0, |
| "completions/max_terminated_length": 12987.0, |
| "completions/mean_length": 9108.96875, |
| "completions/mean_terminated_length": 9108.96875, |
| "completions/min_length": 5579.0, |
| "completions/min_terminated_length": 5579.0, |
| "entropy": 0.29258614405989647, |
| "epoch": 0.060024009603841535, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4107678234577179, |
| "learning_rate": 8.04e-07, |
| "loss": -0.0154, |
| "num_tokens": 18501752.0, |
| "reward": 1.3277008533477783, |
| "reward_std": 2.548473596572876, |
| "rewards/reward_func/mean": 1.3277008533477783, |
| "rewards/reward_func/std": 2.548473358154297, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.7192769050598145, |
| "sampling/importance_sampling_ratio/mean": 0.9897809624671936, |
| "sampling/importance_sampling_ratio/min": 0.033950600773096085, |
| "sampling/sampling_logp_difference/max": 3.3828487396240234, |
| "sampling/sampling_logp_difference/mean": 0.020536189898848534, |
| "step": 50, |
| "step_time": 202.79201423272025 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15226.0, |
| "completions/max_terminated_length": 15226.0, |
| "completions/mean_length": 8895.78125, |
| "completions/mean_terminated_length": 8895.78125, |
| "completions/min_length": 3790.0, |
| "completions/min_terminated_length": 3790.0, |
| "entropy": 0.24668313097208738, |
| "epoch": 0.061224489795918366, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4362691640853882, |
| "learning_rate": 8e-07, |
| "loss": -0.015, |
| "num_tokens": 18819793.0, |
| "reward": 4.881908893585205, |
| "reward_std": 4.377172946929932, |
| "rewards/reward_func/mean": 4.881908893585205, |
| "rewards/reward_func/std": 4.377172470092773, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1437125205993652, |
| "sampling/importance_sampling_ratio/mean": 0.991713285446167, |
| "sampling/importance_sampling_ratio/min": 0.0913717970252037, |
| "sampling/sampling_logp_difference/max": 2.3928184509277344, |
| "sampling/sampling_logp_difference/mean": 0.0172872394323349, |
| "step": 51, |
| "step_time": 230.47769821400288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10907.0, |
| "completions/max_terminated_length": 10907.0, |
| "completions/mean_length": 8155.625, |
| "completions/mean_terminated_length": 8155.625, |
| "completions/min_length": 4708.0, |
| "completions/min_terminated_length": 4708.0, |
| "entropy": 0.2947898115962744, |
| "epoch": 0.062424969987995196, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3665950894355774, |
| "learning_rate": 7.96e-07, |
| "loss": 0.0084, |
| "num_tokens": 19104357.0, |
| "reward": 2.1094980239868164, |
| "reward_std": 2.361452102661133, |
| "rewards/reward_func/mean": 2.1094980239868164, |
| "rewards/reward_func/std": 2.361452102661133, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.565842866897583, |
| "sampling/importance_sampling_ratio/mean": 0.9897834062576294, |
| "sampling/importance_sampling_ratio/min": 0.222982257604599, |
| "sampling/sampling_logp_difference/max": 1.500663161277771, |
| "sampling/sampling_logp_difference/mean": 0.02075599506497383, |
| "step": 52, |
| "step_time": 177.9754405770218 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19155.0, |
| "completions/max_terminated_length": 19155.0, |
| "completions/mean_length": 10726.125, |
| "completions/mean_terminated_length": 10726.125, |
| "completions/min_length": 5744.0, |
| "completions/min_terminated_length": 5744.0, |
| "entropy": 0.2794426204636693, |
| "epoch": 0.06362545018007203, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.34014397859573364, |
| "learning_rate": 7.92e-07, |
| "loss": 0.051, |
| "num_tokens": 19475873.0, |
| "reward": 0.08617197722196579, |
| "reward_std": 0.35582923889160156, |
| "rewards/reward_func/mean": 0.10731302201747894, |
| "rewards/reward_func/std": 0.3313770294189453, |
| "rewards/soft_overlong_punishment_reward/mean": -0.02114105224609375, |
| "rewards/soft_overlong_punishment_reward/std": 0.1195918545126915, |
| "sampling/importance_sampling_ratio/max": 2.9801175594329834, |
| "sampling/importance_sampling_ratio/mean": 0.990292489528656, |
| "sampling/importance_sampling_ratio/min": 0.1588689088821411, |
| "sampling/sampling_logp_difference/max": 1.8396759033203125, |
| "sampling/sampling_logp_difference/mean": 0.019603673368692398, |
| "step": 53, |
| "step_time": 264.9613647001097 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14962.0, |
| "completions/max_terminated_length": 14962.0, |
| "completions/mean_length": 10665.8125, |
| "completions/mean_terminated_length": 10665.8125, |
| "completions/min_length": 5778.0, |
| "completions/min_terminated_length": 5778.0, |
| "entropy": 0.2630910240113735, |
| "epoch": 0.06482593037214886, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36794134974479675, |
| "learning_rate": 7.88e-07, |
| "loss": 0.0013, |
| "num_tokens": 19851875.0, |
| "reward": 1.1953718662261963, |
| "reward_std": 2.0580642223358154, |
| "rewards/reward_func/mean": 1.1953718662261963, |
| "rewards/reward_func/std": 2.0580642223358154, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.635934829711914, |
| "sampling/importance_sampling_ratio/mean": 0.9908843040466309, |
| "sampling/importance_sampling_ratio/min": 0.25040531158447266, |
| "sampling/sampling_logp_difference/max": 1.3846744298934937, |
| "sampling/sampling_logp_difference/mean": 0.01858539693057537, |
| "step": 54, |
| "step_time": 230.69495908927638 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13381.0, |
| "completions/max_terminated_length": 13381.0, |
| "completions/mean_length": 8781.25, |
| "completions/mean_terminated_length": 8781.25, |
| "completions/min_length": 3175.0, |
| "completions/min_terminated_length": 3175.0, |
| "entropy": 0.2748223738744855, |
| "epoch": 0.06602641056422569, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42096376419067383, |
| "learning_rate": 7.84e-07, |
| "loss": -0.0035, |
| "num_tokens": 20161883.0, |
| "reward": 0.9055551290512085, |
| "reward_std": 0.9041584134101868, |
| "rewards/reward_func/mean": 0.9055551290512085, |
| "rewards/reward_func/std": 0.9041584730148315, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9826617240905762, |
| "sampling/importance_sampling_ratio/mean": 0.9905050992965698, |
| "sampling/importance_sampling_ratio/min": 0.18111246824264526, |
| "sampling/sampling_logp_difference/max": 1.708636999130249, |
| "sampling/sampling_logp_difference/mean": 0.01938163861632347, |
| "step": 55, |
| "step_time": 206.5735895021353 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12658.0, |
| "completions/max_terminated_length": 12658.0, |
| "completions/mean_length": 8712.09375, |
| "completions/mean_terminated_length": 8712.09375, |
| "completions/min_length": 5680.0, |
| "completions/min_terminated_length": 5680.0, |
| "entropy": 0.2725402005016804, |
| "epoch": 0.06722689075630252, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4232986569404602, |
| "learning_rate": 7.799999999999999e-07, |
| "loss": 0.0346, |
| "num_tokens": 20467166.0, |
| "reward": 1.345833659172058, |
| "reward_std": 2.5527396202087402, |
| "rewards/reward_func/mean": 1.345833659172058, |
| "rewards/reward_func/std": 2.552739381790161, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3792266845703125, |
| "sampling/importance_sampling_ratio/mean": 0.9905150532722473, |
| "sampling/importance_sampling_ratio/min": 0.2472153902053833, |
| "sampling/sampling_logp_difference/max": 1.3974952697753906, |
| "sampling/sampling_logp_difference/mean": 0.01930988021194935, |
| "step": 56, |
| "step_time": 218.33462092967238 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12566.0, |
| "completions/max_terminated_length": 12566.0, |
| "completions/mean_length": 9388.78125, |
| "completions/mean_terminated_length": 9388.78125, |
| "completions/min_length": 6542.0, |
| "completions/min_terminated_length": 6542.0, |
| "entropy": 0.2632417492568493, |
| "epoch": 0.06842737094837935, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3933756649494171, |
| "learning_rate": 7.76e-07, |
| "loss": -0.0044, |
| "num_tokens": 20799727.0, |
| "reward": 0.655514121055603, |
| "reward_std": 1.1067219972610474, |
| "rewards/reward_func/mean": 0.655514121055603, |
| "rewards/reward_func/std": 1.1067217588424683, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9161028861999512, |
| "sampling/importance_sampling_ratio/mean": 0.9906187653541565, |
| "sampling/importance_sampling_ratio/min": 0.03224591165781021, |
| "sampling/sampling_logp_difference/max": 3.434364080429077, |
| "sampling/sampling_logp_difference/mean": 0.018842609599232674, |
| "step": 57, |
| "step_time": 200.1914525568718 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13388.0, |
| "completions/max_terminated_length": 13388.0, |
| "completions/mean_length": 9069.125, |
| "completions/mean_terminated_length": 9069.125, |
| "completions/min_length": 3500.0, |
| "completions/min_terminated_length": 3500.0, |
| "entropy": 0.26544647198170424, |
| "epoch": 0.06962785114045618, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.34516435861587524, |
| "learning_rate": 7.72e-07, |
| "loss": 0.028, |
| "num_tokens": 21118195.0, |
| "reward": 1.8895032405853271, |
| "reward_std": 2.862610101699829, |
| "rewards/reward_func/mean": 1.8895032405853271, |
| "rewards/reward_func/std": 2.862610101699829, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9907073378562927, |
| "sampling/importance_sampling_ratio/min": 0.20175053179264069, |
| "sampling/sampling_logp_difference/max": 1.6007232666015625, |
| "sampling/sampling_logp_difference/mean": 0.0191123615950346, |
| "step": 58, |
| "step_time": 416.02178934798576 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16766.0, |
| "completions/max_terminated_length": 16766.0, |
| "completions/mean_length": 9156.90625, |
| "completions/mean_terminated_length": 9156.90625, |
| "completions/min_length": 6183.0, |
| "completions/min_terminated_length": 6183.0, |
| "entropy": 0.2845827713608742, |
| "epoch": 0.07082833133253301, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4064350128173828, |
| "learning_rate": 7.68e-07, |
| "loss": -0.0212, |
| "num_tokens": 21436904.0, |
| "reward": 3.7760183811187744, |
| "reward_std": 4.1162896156311035, |
| "rewards/reward_func/mean": 3.778932809829712, |
| "rewards/reward_func/std": 4.113493919372559, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0029144287109375, |
| "rewards/soft_overlong_punishment_reward/std": 0.016486497595906258, |
| "sampling/importance_sampling_ratio/max": 2.561572790145874, |
| "sampling/importance_sampling_ratio/mean": 0.9900461435317993, |
| "sampling/importance_sampling_ratio/min": 0.3691723942756653, |
| "sampling/sampling_logp_difference/max": 0.996491551399231, |
| "sampling/sampling_logp_difference/mean": 0.020054107531905174, |
| "step": 59, |
| "step_time": 232.3352104361402 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15562.0, |
| "completions/max_terminated_length": 15562.0, |
| "completions/mean_length": 8621.25, |
| "completions/mean_terminated_length": 8621.25, |
| "completions/min_length": 3889.0, |
| "completions/min_terminated_length": 3889.0, |
| "entropy": 0.2679576650261879, |
| "epoch": 0.07202881152460984, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4170191287994385, |
| "learning_rate": 7.64e-07, |
| "loss": -0.014, |
| "num_tokens": 21749208.0, |
| "reward": 2.277270555496216, |
| "reward_std": 3.568782329559326, |
| "rewards/reward_func/mean": 2.277270555496216, |
| "rewards/reward_func/std": 3.568782329559326, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.22825288772583, |
| "sampling/importance_sampling_ratio/mean": 0.9907122254371643, |
| "sampling/importance_sampling_ratio/min": 0.2719807028770447, |
| "sampling/sampling_logp_difference/max": 1.302024245262146, |
| "sampling/sampling_logp_difference/mean": 0.019024543464183807, |
| "step": 60, |
| "step_time": 241.59232664620504 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16602.0, |
| "completions/max_terminated_length": 16602.0, |
| "completions/mean_length": 11490.8125, |
| "completions/mean_terminated_length": 11490.8125, |
| "completions/min_length": 7465.0, |
| "completions/min_terminated_length": 7465.0, |
| "entropy": 0.24971914291381836, |
| "epoch": 0.07322929171668667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36966225504875183, |
| "learning_rate": 7.599999999999999e-07, |
| "loss": 0.0258, |
| "num_tokens": 22150906.0, |
| "reward": 2.009342670440674, |
| "reward_std": 2.9576900005340576, |
| "rewards/reward_func/mean": 2.0110058784484863, |
| "rewards/reward_func/std": 2.956507444381714, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0016632080078125, |
| "rewards/soft_overlong_punishment_reward/std": 0.009408525191247463, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9912000894546509, |
| "sampling/importance_sampling_ratio/min": 0.0796288475394249, |
| "sampling/sampling_logp_difference/max": 2.530378818511963, |
| "sampling/sampling_logp_difference/mean": 0.018482467159628868, |
| "step": 61, |
| "step_time": 262.58794342912734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16208.0, |
| "completions/max_terminated_length": 16208.0, |
| "completions/mean_length": 9912.53125, |
| "completions/mean_terminated_length": 9912.53125, |
| "completions/min_length": 3451.0, |
| "completions/min_terminated_length": 3451.0, |
| "entropy": 0.2602381184697151, |
| "epoch": 0.0744297719087635, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.45621025562286377, |
| "learning_rate": 7.559999999999999e-07, |
| "loss": -0.005, |
| "num_tokens": 22506499.0, |
| "reward": 0.5048784017562866, |
| "reward_std": 0.6571192145347595, |
| "rewards/reward_func/mean": 0.5048784017562866, |
| "rewards/reward_func/std": 0.6571192145347595, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.990957498550415, |
| "sampling/importance_sampling_ratio/min": 0.23703482747077942, |
| "sampling/sampling_logp_difference/max": 1.4395482540130615, |
| "sampling/sampling_logp_difference/mean": 0.01870187371969223, |
| "step": 62, |
| "step_time": 227.91837301501073 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13957.0, |
| "completions/max_terminated_length": 13957.0, |
| "completions/mean_length": 10636.0625, |
| "completions/mean_terminated_length": 10636.0625, |
| "completions/min_length": 8297.0, |
| "completions/min_terminated_length": 8297.0, |
| "entropy": 0.24767167586833239, |
| "epoch": 0.07563025210084033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3626573383808136, |
| "learning_rate": 7.52e-07, |
| "loss": 0.0097, |
| "num_tokens": 22889037.0, |
| "reward": 2.379978656768799, |
| "reward_std": 2.5725669860839844, |
| "rewards/reward_func/mean": 2.379978656768799, |
| "rewards/reward_func/std": 2.5725667476654053, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4255101680755615, |
| "sampling/importance_sampling_ratio/mean": 0.991312563419342, |
| "sampling/importance_sampling_ratio/min": 0.31686195731163025, |
| "sampling/sampling_logp_difference/max": 1.1492891311645508, |
| "sampling/sampling_logp_difference/mean": 0.017962327226996422, |
| "step": 63, |
| "step_time": 220.6857718299143 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14398.0, |
| "completions/max_terminated_length": 14398.0, |
| "completions/mean_length": 10088.125, |
| "completions/mean_terminated_length": 10088.125, |
| "completions/min_length": 6941.0, |
| "completions/min_terminated_length": 6941.0, |
| "entropy": 0.25979764945805073, |
| "epoch": 0.07683073229291716, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3685627281665802, |
| "learning_rate": 7.48e-07, |
| "loss": 0.0522, |
| "num_tokens": 23250569.0, |
| "reward": 3681.080322265625, |
| "reward_std": 20817.853515625, |
| "rewards/reward_func/mean": 3681.080322265625, |
| "rewards/reward_func/std": 20817.853515625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9908851981163025, |
| "sampling/importance_sampling_ratio/min": 0.15633250772953033, |
| "sampling/sampling_logp_difference/max": 1.8557701110839844, |
| "sampling/sampling_logp_difference/mean": 0.01882169209420681, |
| "step": 64, |
| "step_time": 207.90169405727647 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15345.0, |
| "completions/max_terminated_length": 15345.0, |
| "completions/mean_length": 10208.5625, |
| "completions/mean_terminated_length": 10208.5625, |
| "completions/min_length": 4160.0, |
| "completions/min_terminated_length": 4160.0, |
| "entropy": 0.25587698817253113, |
| "epoch": 0.07803121248499399, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37043145298957825, |
| "learning_rate": 7.44e-07, |
| "loss": -0.0128, |
| "num_tokens": 23616123.0, |
| "reward": 115.96958923339844, |
| "reward_std": 450.455322265625, |
| "rewards/reward_func/mean": 115.96958923339844, |
| "rewards/reward_func/std": 450.455322265625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3009722232818604, |
| "sampling/importance_sampling_ratio/mean": 0.9911485314369202, |
| "sampling/importance_sampling_ratio/min": 0.3561326265335083, |
| "sampling/sampling_logp_difference/max": 1.03245210647583, |
| "sampling/sampling_logp_difference/mean": 0.018131505697965622, |
| "step": 65, |
| "step_time": 247.2077819444239 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15775.0, |
| "completions/max_terminated_length": 15775.0, |
| "completions/mean_length": 10089.09375, |
| "completions/mean_terminated_length": 10089.09375, |
| "completions/min_length": 5130.0, |
| "completions/min_terminated_length": 5130.0, |
| "entropy": 0.25899219047278166, |
| "epoch": 0.07923169267707082, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38848623633384705, |
| "learning_rate": 7.4e-07, |
| "loss": -0.0311, |
| "num_tokens": 23972966.0, |
| "reward": 2.4493942260742188, |
| "reward_std": 3.161067247390747, |
| "rewards/reward_func/mean": 2.4493942260742188, |
| "rewards/reward_func/std": 3.161067247390747, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.568775177001953, |
| "sampling/importance_sampling_ratio/mean": 0.9909378290176392, |
| "sampling/importance_sampling_ratio/min": 0.3397059142589569, |
| "sampling/sampling_logp_difference/max": 1.0796749591827393, |
| "sampling/sampling_logp_difference/mean": 0.018584787845611572, |
| "step": 66, |
| "step_time": 235.83589306659997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17692.0, |
| "completions/max_terminated_length": 17692.0, |
| "completions/mean_length": 11849.34375, |
| "completions/mean_terminated_length": 11849.34375, |
| "completions/min_length": 6275.0, |
| "completions/min_terminated_length": 6275.0, |
| "entropy": 0.2503598555922508, |
| "epoch": 0.08043217286914765, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36103734374046326, |
| "learning_rate": 7.359999999999999e-07, |
| "loss": 0.0653, |
| "num_tokens": 24390121.0, |
| "reward": 0.5631309747695923, |
| "reward_std": 0.7355284690856934, |
| "rewards/reward_func/mean": 0.5829674005508423, |
| "rewards/reward_func/std": 0.7148671746253967, |
| "rewards/soft_overlong_punishment_reward/mean": -0.01983642578125, |
| "rewards/soft_overlong_punishment_reward/std": 0.07805701345205307, |
| "sampling/importance_sampling_ratio/max": 2.5908403396606445, |
| "sampling/importance_sampling_ratio/mean": 0.9912521243095398, |
| "sampling/importance_sampling_ratio/min": 0.20392736792564392, |
| "sampling/sampling_logp_difference/max": 1.5899913311004639, |
| "sampling/sampling_logp_difference/mean": 0.017768753692507744, |
| "step": 67, |
| "step_time": 273.55189195496496 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13621.0, |
| "completions/max_terminated_length": 13621.0, |
| "completions/mean_length": 10604.21875, |
| "completions/mean_terminated_length": 10604.21875, |
| "completions/min_length": 7394.0, |
| "completions/min_terminated_length": 7394.0, |
| "entropy": 0.2723994627594948, |
| "epoch": 0.08163265306122448, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40947461128234863, |
| "learning_rate": 7.319999999999999e-07, |
| "loss": 0.017, |
| "num_tokens": 24763560.0, |
| "reward": 15.798975944519043, |
| "reward_std": 37.360469818115234, |
| "rewards/reward_func/mean": 15.798975944519043, |
| "rewards/reward_func/std": 37.360469818115234, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.846003293991089, |
| "sampling/importance_sampling_ratio/mean": 0.9903964996337891, |
| "sampling/importance_sampling_ratio/min": 0.025304390117526054, |
| "sampling/sampling_logp_difference/max": 3.6767773628234863, |
| "sampling/sampling_logp_difference/mean": 0.01946277916431427, |
| "step": 68, |
| "step_time": 208.66941105970182 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12376.0, |
| "completions/max_terminated_length": 12376.0, |
| "completions/mean_length": 9659.3125, |
| "completions/mean_terminated_length": 9659.3125, |
| "completions/min_length": 7514.0, |
| "completions/min_terminated_length": 7514.0, |
| "entropy": 0.2647173348814249, |
| "epoch": 0.08283313325330131, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3932405710220337, |
| "learning_rate": 7.28e-07, |
| "loss": -0.0129, |
| "num_tokens": 25105250.0, |
| "reward": 1.1635289192199707, |
| "reward_std": 1.419382929801941, |
| "rewards/reward_func/mean": 1.1635289192199707, |
| "rewards/reward_func/std": 1.4193830490112305, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.187065362930298, |
| "sampling/importance_sampling_ratio/mean": 0.9907566905021667, |
| "sampling/importance_sampling_ratio/min": 0.29801252484321594, |
| "sampling/sampling_logp_difference/max": 1.2106198072433472, |
| "sampling/sampling_logp_difference/mean": 0.018872233107686043, |
| "step": 69, |
| "step_time": 308.8531298188027 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15954.0, |
| "completions/max_terminated_length": 15954.0, |
| "completions/mean_length": 10261.5625, |
| "completions/mean_terminated_length": 10261.5625, |
| "completions/min_length": 6635.0, |
| "completions/min_terminated_length": 6635.0, |
| "entropy": 0.25401466619223356, |
| "epoch": 0.08403361344537816, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.32909703254699707, |
| "learning_rate": 7.24e-07, |
| "loss": 0.0189, |
| "num_tokens": 25472484.0, |
| "reward": 0.7801880836486816, |
| "reward_std": 0.7046358585357666, |
| "rewards/reward_func/mean": 0.7801880836486816, |
| "rewards/reward_func/std": 0.7046358585357666, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9910746812820435, |
| "sampling/importance_sampling_ratio/min": 0.15575462579727173, |
| "sampling/sampling_logp_difference/max": 1.8594733476638794, |
| "sampling/sampling_logp_difference/mean": 0.018227702006697655, |
| "step": 70, |
| "step_time": 458.7579943599412 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17229.0, |
| "completions/max_terminated_length": 17229.0, |
| "completions/mean_length": 11900.3125, |
| "completions/mean_terminated_length": 11900.3125, |
| "completions/min_length": 8069.0, |
| "completions/min_terminated_length": 8069.0, |
| "entropy": 0.23596489429473877, |
| "epoch": 0.08523409363745499, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.28990110754966736, |
| "learning_rate": 7.2e-07, |
| "loss": 0.0406, |
| "num_tokens": 25902358.0, |
| "reward": 1.1131587028503418, |
| "reward_std": 2.4644935131073, |
| "rewards/reward_func/mean": 1.119605541229248, |
| "rewards/reward_func/std": 2.461198329925537, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00644683837890625, |
| "rewards/soft_overlong_punishment_reward/std": 0.03646882623434067, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9916808605194092, |
| "sampling/importance_sampling_ratio/min": 0.07585223764181137, |
| "sampling/sampling_logp_difference/max": 2.578968048095703, |
| "sampling/sampling_logp_difference/mean": 0.017171792685985565, |
| "step": 71, |
| "step_time": 256.0422782639507 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15647.0, |
| "completions/max_terminated_length": 15647.0, |
| "completions/mean_length": 10677.8125, |
| "completions/mean_terminated_length": 10677.8125, |
| "completions/min_length": 6661.0, |
| "completions/min_terminated_length": 6661.0, |
| "entropy": 0.24026651121675968, |
| "epoch": 0.08643457382953182, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36302992701530457, |
| "learning_rate": 7.159999999999999e-07, |
| "loss": 0.0069, |
| "num_tokens": 26286872.0, |
| "reward": 6.745024681091309, |
| "reward_std": 27.16277313232422, |
| "rewards/reward_func/mean": 6.745024681091309, |
| "rewards/reward_func/std": 27.162776947021484, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.476977825164795, |
| "sampling/importance_sampling_ratio/mean": 0.991825520992279, |
| "sampling/importance_sampling_ratio/min": 0.006692860741168261, |
| "sampling/sampling_logp_difference/max": 5.0067138671875, |
| "sampling/sampling_logp_difference/mean": 0.01745101995766163, |
| "step": 72, |
| "step_time": 454.2569543699501 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14315.0, |
| "completions/max_terminated_length": 14315.0, |
| "completions/mean_length": 6956.90625, |
| "completions/mean_terminated_length": 6956.90625, |
| "completions/min_length": 3062.0, |
| "completions/min_terminated_length": 3062.0, |
| "entropy": 0.28412431478500366, |
| "epoch": 0.08763505402160865, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5102128386497498, |
| "learning_rate": 7.119999999999999e-07, |
| "loss": 0.0493, |
| "num_tokens": 26533853.0, |
| "reward": 1.6690137386322021, |
| "reward_std": 2.0055153369903564, |
| "rewards/reward_func/mean": 1.6690137386322021, |
| "rewards/reward_func/std": 2.0055153369903564, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9905214905738831, |
| "sampling/importance_sampling_ratio/min": 0.027704769745469093, |
| "sampling/sampling_logp_difference/max": 3.586150646209717, |
| "sampling/sampling_logp_difference/mean": 0.019451972097158432, |
| "step": 73, |
| "step_time": 203.57053557073232 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11038.0, |
| "completions/max_terminated_length": 11038.0, |
| "completions/mean_length": 7485.53125, |
| "completions/mean_terminated_length": 7485.53125, |
| "completions/min_length": 3940.0, |
| "completions/min_terminated_length": 3940.0, |
| "entropy": 0.27887804340571165, |
| "epoch": 0.08883553421368548, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3595903813838959, |
| "learning_rate": 7.079999999999999e-07, |
| "loss": 0.0016, |
| "num_tokens": 26797934.0, |
| "reward": 0.582500159740448, |
| "reward_std": 0.6758586764335632, |
| "rewards/reward_func/mean": 0.582500159740448, |
| "rewards/reward_func/std": 0.6758586168289185, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3009722232818604, |
| "sampling/importance_sampling_ratio/mean": 0.9900974035263062, |
| "sampling/importance_sampling_ratio/min": 0.3130251169204712, |
| "sampling/sampling_logp_difference/max": 1.1614718437194824, |
| "sampling/sampling_logp_difference/mean": 0.01946306601166725, |
| "step": 74, |
| "step_time": 394.7669193340698 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12980.0, |
| "completions/max_terminated_length": 12980.0, |
| "completions/mean_length": 9189.96875, |
| "completions/mean_terminated_length": 9189.96875, |
| "completions/min_length": 4670.0, |
| "completions/min_terminated_length": 4670.0, |
| "entropy": 0.27417559176683426, |
| "epoch": 0.09003601440576231, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41591066122055054, |
| "learning_rate": 7.04e-07, |
| "loss": 0.0263, |
| "num_tokens": 27116477.0, |
| "reward": 1.5546950101852417, |
| "reward_std": 2.706759452819824, |
| "rewards/reward_func/mean": 1.5546950101852417, |
| "rewards/reward_func/std": 2.706759214401245, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9904885292053223, |
| "sampling/importance_sampling_ratio/min": 0.23027251660823822, |
| "sampling/sampling_logp_difference/max": 1.468491792678833, |
| "sampling/sampling_logp_difference/mean": 0.019385498017072678, |
| "step": 75, |
| "step_time": 212.57274886220694 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15570.0, |
| "completions/max_terminated_length": 15570.0, |
| "completions/mean_length": 9589.8125, |
| "completions/mean_terminated_length": 9589.8125, |
| "completions/min_length": 3431.0, |
| "completions/min_terminated_length": 3431.0, |
| "entropy": 0.24109713081270456, |
| "epoch": 0.09123649459783914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39267539978027344, |
| "learning_rate": 7e-07, |
| "loss": -0.0101, |
| "num_tokens": 27469983.0, |
| "reward": 1.6705238819122314, |
| "reward_std": 1.578829288482666, |
| "rewards/reward_func/mean": 1.6705238819122314, |
| "rewards/reward_func/std": 1.578829288482666, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9916650056838989, |
| "sampling/importance_sampling_ratio/min": 0.3055680990219116, |
| "sampling/sampling_logp_difference/max": 1.5075340270996094, |
| "sampling/sampling_logp_difference/mean": 0.01719709485769272, |
| "step": 76, |
| "step_time": 434.341842096881 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12644.0, |
| "completions/max_terminated_length": 12644.0, |
| "completions/mean_length": 8874.84375, |
| "completions/mean_terminated_length": 8874.84375, |
| "completions/min_length": 4780.0, |
| "completions/min_terminated_length": 4780.0, |
| "entropy": 0.28068294655531645, |
| "epoch": 0.09243697478991597, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4375174045562744, |
| "learning_rate": 6.959999999999999e-07, |
| "loss": 0.0539, |
| "num_tokens": 27781946.0, |
| "reward": 19.96912956237793, |
| "reward_std": 36.33182144165039, |
| "rewards/reward_func/mean": 19.96912956237793, |
| "rewards/reward_func/std": 36.33182144165039, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.198267698287964, |
| "sampling/importance_sampling_ratio/mean": 0.990230143070221, |
| "sampling/importance_sampling_ratio/min": 0.1569797843694687, |
| "sampling/sampling_logp_difference/max": 1.8516383171081543, |
| "sampling/sampling_logp_difference/mean": 0.01989983581006527, |
| "step": 77, |
| "step_time": 399.6355443559587 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12663.0, |
| "completions/max_terminated_length": 12663.0, |
| "completions/mean_length": 9384.59375, |
| "completions/mean_terminated_length": 9384.59375, |
| "completions/min_length": 5366.0, |
| "completions/min_terminated_length": 5366.0, |
| "entropy": 0.23270147573202848, |
| "epoch": 0.0936374549819928, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41008490324020386, |
| "learning_rate": 6.919999999999999e-07, |
| "loss": 0.0169, |
| "num_tokens": 28131637.0, |
| "reward": 0.34000474214553833, |
| "reward_std": 0.4872087836265564, |
| "rewards/reward_func/mean": 0.34000474214553833, |
| "rewards/reward_func/std": 0.4872088134288788, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9919241666793823, |
| "sampling/importance_sampling_ratio/min": 0.012431665323674679, |
| "sampling/sampling_logp_difference/max": 4.387508392333984, |
| "sampling/sampling_logp_difference/mean": 0.016751548275351524, |
| "step": 78, |
| "step_time": 213.20301025058143 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13605.0, |
| "completions/max_terminated_length": 13605.0, |
| "completions/mean_length": 9480.09375, |
| "completions/mean_terminated_length": 9480.09375, |
| "completions/min_length": 6576.0, |
| "completions/min_terminated_length": 6576.0, |
| "entropy": 0.2560074757784605, |
| "epoch": 0.09483793517406963, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4534052312374115, |
| "learning_rate": 6.879999999999999e-07, |
| "loss": 0.0027, |
| "num_tokens": 28473544.0, |
| "reward": 1.1451480388641357, |
| "reward_std": 2.02068829536438, |
| "rewards/reward_func/mean": 1.1451480388641357, |
| "rewards/reward_func/std": 2.02068829536438, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9910561442375183, |
| "sampling/importance_sampling_ratio/min": 0.28057095408439636, |
| "sampling/sampling_logp_difference/max": 1.2709286212921143, |
| "sampling/sampling_logp_difference/mean": 0.018182622268795967, |
| "step": 79, |
| "step_time": 198.80667186016217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14034.0, |
| "completions/max_terminated_length": 14034.0, |
| "completions/mean_length": 10575.3125, |
| "completions/mean_terminated_length": 10575.3125, |
| "completions/min_length": 7172.0, |
| "completions/min_terminated_length": 7172.0, |
| "entropy": 0.25964460615068674, |
| "epoch": 0.09603841536614646, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4087451696395874, |
| "learning_rate": 6.84e-07, |
| "loss": 0.0462, |
| "num_tokens": 28857042.0, |
| "reward": 0.8121464252471924, |
| "reward_std": 1.5952792167663574, |
| "rewards/reward_func/mean": 0.8121464252471924, |
| "rewards/reward_func/std": 1.595279335975647, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.889493227005005, |
| "sampling/importance_sampling_ratio/mean": 0.9910570383071899, |
| "sampling/importance_sampling_ratio/min": 0.0975920781493187, |
| "sampling/sampling_logp_difference/max": 2.3269588947296143, |
| "sampling/sampling_logp_difference/mean": 0.01859731785953045, |
| "step": 80, |
| "step_time": 237.49234476010315 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17704.0, |
| "completions/max_terminated_length": 17704.0, |
| "completions/mean_length": 10183.125, |
| "completions/mean_terminated_length": 10183.125, |
| "completions/min_length": 5525.0, |
| "completions/min_terminated_length": 5525.0, |
| "entropy": 0.25165214855223894, |
| "epoch": 0.09723889555822329, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3784518241882324, |
| "learning_rate": 6.800000000000001e-07, |
| "loss": -0.0141, |
| "num_tokens": 29213526.0, |
| "reward": 8.427579879760742, |
| "reward_std": 24.21912956237793, |
| "rewards/reward_func/mean": 8.437650680541992, |
| "rewards/reward_func/std": 24.21601676940918, |
| "rewards/soft_overlong_punishment_reward/mean": -0.01007080078125, |
| "rewards/soft_overlong_punishment_reward/std": 0.0569690503180027, |
| "sampling/importance_sampling_ratio/max": 2.1546497344970703, |
| "sampling/importance_sampling_ratio/mean": 0.9912103414535522, |
| "sampling/importance_sampling_ratio/min": 0.3988558351993561, |
| "sampling/sampling_logp_difference/max": 0.9191552400588989, |
| "sampling/sampling_logp_difference/mean": 0.018073324114084244, |
| "step": 81, |
| "step_time": 258.5655134237604 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12930.0, |
| "completions/max_terminated_length": 12930.0, |
| "completions/mean_length": 8081.375, |
| "completions/mean_terminated_length": 8081.375, |
| "completions/min_length": 4585.0, |
| "completions/min_terminated_length": 4585.0, |
| "entropy": 0.2729193214327097, |
| "epoch": 0.09843937575030012, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43891701102256775, |
| "learning_rate": 6.76e-07, |
| "loss": 0.0029, |
| "num_tokens": 29504138.0, |
| "reward": 2.3440794944763184, |
| "reward_std": 2.2432892322540283, |
| "rewards/reward_func/mean": 2.3440794944763184, |
| "rewards/reward_func/std": 2.2432892322540283, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9903400540351868, |
| "sampling/importance_sampling_ratio/min": 0.15401123464107513, |
| "sampling/sampling_logp_difference/max": 1.8707298040390015, |
| "sampling/sampling_logp_difference/mean": 0.01947890594601631, |
| "step": 82, |
| "step_time": 202.33901768375654 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14334.0, |
| "completions/max_terminated_length": 14334.0, |
| "completions/mean_length": 8477.21875, |
| "completions/mean_terminated_length": 8477.21875, |
| "completions/min_length": 2572.0, |
| "completions/min_terminated_length": 2572.0, |
| "entropy": 0.26816452480852604, |
| "epoch": 0.09963985594237695, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.48063552379608154, |
| "learning_rate": 6.72e-07, |
| "loss": 0.0714, |
| "num_tokens": 29807561.0, |
| "reward": 2.8314692974090576, |
| "reward_std": 3.5692901611328125, |
| "rewards/reward_func/mean": 2.8314692974090576, |
| "rewards/reward_func/std": 3.5692899227142334, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.565842866897583, |
| "sampling/importance_sampling_ratio/mean": 0.9907183051109314, |
| "sampling/importance_sampling_ratio/min": 0.19031807780265808, |
| "sampling/sampling_logp_difference/max": 1.6590585708618164, |
| "sampling/sampling_logp_difference/mean": 0.018856924027204514, |
| "step": 83, |
| "step_time": 217.18472903524525 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17204.0, |
| "completions/max_terminated_length": 17204.0, |
| "completions/mean_length": 8830.375, |
| "completions/mean_terminated_length": 8830.375, |
| "completions/min_length": 4996.0, |
| "completions/min_terminated_length": 4996.0, |
| "entropy": 0.27092269994318485, |
| "epoch": 0.10084033613445378, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41215991973876953, |
| "learning_rate": 6.68e-07, |
| "loss": -0.0409, |
| "num_tokens": 30123829.0, |
| "reward": 2.2067344188690186, |
| "reward_std": 3.047388792037964, |
| "rewards/reward_func/mean": 2.2129905223846436, |
| "rewards/reward_func/std": 3.0453271865844727, |
| "rewards/soft_overlong_punishment_reward/mean": -0.006256103515625, |
| "rewards/soft_overlong_punishment_reward/std": 0.03538986295461655, |
| "sampling/importance_sampling_ratio/max": 2.052222490310669, |
| "sampling/importance_sampling_ratio/mean": 0.990820586681366, |
| "sampling/importance_sampling_ratio/min": 0.3114754259586334, |
| "sampling/sampling_logp_difference/max": 1.1664347648620605, |
| "sampling/sampling_logp_difference/mean": 0.0186629518866539, |
| "step": 84, |
| "step_time": 211.40592918510083 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12600.0, |
| "completions/max_terminated_length": 12600.0, |
| "completions/mean_length": 8298.3125, |
| "completions/mean_terminated_length": 8298.3125, |
| "completions/min_length": 4949.0, |
| "completions/min_terminated_length": 4949.0, |
| "entropy": 0.2829645574092865, |
| "epoch": 0.10204081632653061, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5601394772529602, |
| "learning_rate": 6.64e-07, |
| "loss": 0.0305, |
| "num_tokens": 30421735.0, |
| "reward": 3.7544779777526855, |
| "reward_std": 3.1711204051971436, |
| "rewards/reward_func/mean": 3.7544779777526855, |
| "rewards/reward_func/std": 3.1711206436157227, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901652932167053, |
| "sampling/importance_sampling_ratio/min": 0.12367851287126541, |
| "sampling/sampling_logp_difference/max": 2.0900697708129883, |
| "sampling/sampling_logp_difference/mean": 0.020197300240397453, |
| "step": 85, |
| "step_time": 266.99491723254323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 15285.0, |
| "completions/mean_length": 10550.75, |
| "completions/mean_terminated_length": 10230.451171875, |
| "completions/min_length": 5820.0, |
| "completions/min_terminated_length": 5820.0, |
| "entropy": 0.24555738922208548, |
| "epoch": 0.10324129651860744, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314035564661026, |
| "learning_rate": 6.6e-07, |
| "loss": -0.0973, |
| "num_tokens": 30792879.0, |
| "reward": 1.717525601387024, |
| "reward_std": 3.4412992000579834, |
| "rewards/reward_func/mean": 1.805187702178955, |
| "rewards/reward_func/std": 3.4616737365722656, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9913591742515564, |
| "sampling/importance_sampling_ratio/min": 0.16628147661685944, |
| "sampling/sampling_logp_difference/max": 1.7940733432769775, |
| "sampling/sampling_logp_difference/mean": 0.017975792288780212, |
| "step": 86, |
| "step_time": 266.66299685207196 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13159.0, |
| "completions/max_terminated_length": 13159.0, |
| "completions/mean_length": 9776.40625, |
| "completions/mean_terminated_length": 9776.40625, |
| "completions/min_length": 7210.0, |
| "completions/min_terminated_length": 7210.0, |
| "entropy": 0.24749416019767523, |
| "epoch": 0.10444177671068428, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37279677391052246, |
| "learning_rate": 6.56e-07, |
| "loss": -0.0527, |
| "num_tokens": 31140836.0, |
| "reward": 2.521646738052368, |
| "reward_std": 3.81414794921875, |
| "rewards/reward_func/mean": 2.521646738052368, |
| "rewards/reward_func/std": 3.81414794921875, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4072625637054443, |
| "sampling/importance_sampling_ratio/mean": 0.9911715984344482, |
| "sampling/importance_sampling_ratio/min": 0.19457262754440308, |
| "sampling/sampling_logp_difference/max": 1.6369497776031494, |
| "sampling/sampling_logp_difference/mean": 0.017878280952572823, |
| "step": 87, |
| "step_time": 313.04910568264313 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16942.0, |
| "completions/max_terminated_length": 16942.0, |
| "completions/mean_length": 9632.09375, |
| "completions/mean_terminated_length": 9632.09375, |
| "completions/min_length": 3954.0, |
| "completions/min_terminated_length": 3954.0, |
| "entropy": 0.28111381735652685, |
| "epoch": 0.1056422569027611, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41607415676116943, |
| "learning_rate": 6.52e-07, |
| "loss": 0.0437, |
| "num_tokens": 31474263.0, |
| "reward": 1.1277384757995605, |
| "reward_std": 1.8439369201660156, |
| "rewards/reward_func/mean": 1.131995677947998, |
| "rewards/reward_func/std": 1.8411990404129028, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0042572021484375, |
| "rewards/soft_overlong_punishment_reward/std": 0.024082370102405548, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9902504682540894, |
| "sampling/importance_sampling_ratio/min": 0.01312999241054058, |
| "sampling/sampling_logp_difference/max": 4.332856178283691, |
| "sampling/sampling_logp_difference/mean": 0.01968482695519924, |
| "step": 88, |
| "step_time": 231.826713241986 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16728.0, |
| "completions/max_terminated_length": 16728.0, |
| "completions/mean_length": 10530.4375, |
| "completions/mean_terminated_length": 10530.4375, |
| "completions/min_length": 6271.0, |
| "completions/min_terminated_length": 6271.0, |
| "entropy": 0.25707424618303776, |
| "epoch": 0.10684273709483794, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38471195101737976, |
| "learning_rate": 6.48e-07, |
| "loss": 0.0606, |
| "num_tokens": 31852261.0, |
| "reward": 1.277016282081604, |
| "reward_std": 2.039738893508911, |
| "rewards/reward_func/mean": 1.279640793800354, |
| "rewards/reward_func/std": 2.038050889968872, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00262451171875, |
| "rewards/soft_overlong_punishment_reward/std": 0.014846480451524258, |
| "sampling/importance_sampling_ratio/max": 2.706266164779663, |
| "sampling/importance_sampling_ratio/mean": 0.9911405444145203, |
| "sampling/importance_sampling_ratio/min": 0.22103802859783173, |
| "sampling/sampling_logp_difference/max": 1.5094205141067505, |
| "sampling/sampling_logp_difference/mean": 0.0181552916765213, |
| "step": 89, |
| "step_time": 452.72294589015655 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15561.0, |
| "completions/max_terminated_length": 15561.0, |
| "completions/mean_length": 10859.5, |
| "completions/mean_terminated_length": 10859.5, |
| "completions/min_length": 4952.0, |
| "completions/min_terminated_length": 4952.0, |
| "entropy": 0.24442417547106743, |
| "epoch": 0.10804321728691477, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37132179737091064, |
| "learning_rate": 6.44e-07, |
| "loss": 0.025, |
| "num_tokens": 32237653.0, |
| "reward": 0.5021224617958069, |
| "reward_std": 0.7325555682182312, |
| "rewards/reward_func/mean": 0.5021224617958069, |
| "rewards/reward_func/std": 0.7325555086135864, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3009634017944336, |
| "sampling/importance_sampling_ratio/mean": 0.9916141033172607, |
| "sampling/importance_sampling_ratio/min": 0.0029834972228854895, |
| "sampling/sampling_logp_difference/max": 5.814659118652344, |
| "sampling/sampling_logp_difference/mean": 0.017731234431266785, |
| "step": 90, |
| "step_time": 248.43443835526705 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14837.0, |
| "completions/max_terminated_length": 14837.0, |
| "completions/mean_length": 10019.59375, |
| "completions/mean_terminated_length": 10019.59375, |
| "completions/min_length": 6964.0, |
| "completions/min_terminated_length": 6964.0, |
| "entropy": 0.279639994725585, |
| "epoch": 0.1092436974789916, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3926539421081543, |
| "learning_rate": 6.4e-07, |
| "loss": 0.0568, |
| "num_tokens": 32596080.0, |
| "reward": 2894.755615234375, |
| "reward_std": 7884.30615234375, |
| "rewards/reward_func/mean": 2894.755615234375, |
| "rewards/reward_func/std": 7884.30615234375, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901866316795349, |
| "sampling/importance_sampling_ratio/min": 0.2964800298213959, |
| "sampling/sampling_logp_difference/max": 1.3845736980438232, |
| "sampling/sampling_logp_difference/mean": 0.019792938604950905, |
| "step": 91, |
| "step_time": 230.6299822588917 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12330.0, |
| "completions/max_terminated_length": 12330.0, |
| "completions/mean_length": 9052.78125, |
| "completions/mean_terminated_length": 9052.78125, |
| "completions/min_length": 5906.0, |
| "completions/min_terminated_length": 5906.0, |
| "entropy": 0.31160302832722664, |
| "epoch": 0.11044417767106843, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41976141929626465, |
| "learning_rate": 6.36e-07, |
| "loss": -0.0106, |
| "num_tokens": 32908137.0, |
| "reward": 1.0270764827728271, |
| "reward_std": 1.5837911367416382, |
| "rewards/reward_func/mean": 1.0270764827728271, |
| "rewards/reward_func/std": 1.5837911367416382, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9889901280403137, |
| "sampling/importance_sampling_ratio/min": 0.251454621553421, |
| "sampling/sampling_logp_difference/max": 1.3804928064346313, |
| "sampling/sampling_logp_difference/mean": 0.02152213454246521, |
| "step": 92, |
| "step_time": 209.22184566501528 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13164.0, |
| "completions/max_terminated_length": 13164.0, |
| "completions/mean_length": 7722.125, |
| "completions/mean_terminated_length": 7722.125, |
| "completions/min_length": 3235.0, |
| "completions/min_terminated_length": 3235.0, |
| "entropy": 0.27631482668220997, |
| "epoch": 0.11164465786314526, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43816474080085754, |
| "learning_rate": 6.319999999999999e-07, |
| "loss": -0.0276, |
| "num_tokens": 33179125.0, |
| "reward": 3.121471881866455, |
| "reward_std": 3.0470423698425293, |
| "rewards/reward_func/mean": 3.121471881866455, |
| "rewards/reward_func/std": 3.0470426082611084, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2446866035461426, |
| "sampling/importance_sampling_ratio/mean": 0.990494430065155, |
| "sampling/importance_sampling_ratio/min": 0.3229256570339203, |
| "sampling/sampling_logp_difference/max": 1.1303331851959229, |
| "sampling/sampling_logp_difference/mean": 0.0194493867456913, |
| "step": 93, |
| "step_time": 403.52912656613626 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9834.0, |
| "completions/max_terminated_length": 9834.0, |
| "completions/mean_length": 6429.0, |
| "completions/mean_terminated_length": 6429.0, |
| "completions/min_length": 3298.0, |
| "completions/min_terminated_length": 3298.0, |
| "entropy": 0.3079283908009529, |
| "epoch": 0.11284513805522209, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.512400209903717, |
| "learning_rate": 6.28e-07, |
| "loss": -0.0118, |
| "num_tokens": 33409485.0, |
| "reward": 4.05039119720459, |
| "reward_std": 6.237717628479004, |
| "rewards/reward_func/mean": 4.05039119720459, |
| "rewards/reward_func/std": 6.237717151641846, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9068455696105957, |
| "sampling/importance_sampling_ratio/mean": 0.9891414046287537, |
| "sampling/importance_sampling_ratio/min": 0.28006723523139954, |
| "sampling/sampling_logp_difference/max": 1.2727255821228027, |
| "sampling/sampling_logp_difference/mean": 0.021337833255529404, |
| "step": 94, |
| "step_time": 150.58229626482353 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 13641.0, |
| "completions/mean_length": 8809.875, |
| "completions/mean_terminated_length": 8433.4189453125, |
| "completions/min_length": 2494.0, |
| "completions/min_terminated_length": 2494.0, |
| "entropy": 0.28533732518553734, |
| "epoch": 0.11404561824729892, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34757664799690247, |
| "learning_rate": 6.24e-07, |
| "loss": -0.1059, |
| "num_tokens": 33722393.0, |
| "reward": 2.1445155143737793, |
| "reward_std": 2.832806348800659, |
| "rewards/reward_func/mean": 2.2459514141082764, |
| "rewards/reward_func/std": 2.8199384212493896, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901010990142822, |
| "sampling/importance_sampling_ratio/min": 0.0004452175635378808, |
| "sampling/sampling_logp_difference/max": 7.716947555541992, |
| "sampling/sampling_logp_difference/mean": 0.01991894841194153, |
| "step": 95, |
| "step_time": 253.43176098098047 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15697.0, |
| "completions/max_terminated_length": 15697.0, |
| "completions/mean_length": 10009.0625, |
| "completions/mean_terminated_length": 10009.0625, |
| "completions/min_length": 5990.0, |
| "completions/min_terminated_length": 5990.0, |
| "entropy": 0.24590188823640347, |
| "epoch": 0.11524609843937575, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.316346138715744, |
| "learning_rate": 6.2e-07, |
| "loss": -0.0135, |
| "num_tokens": 34078643.0, |
| "reward": 0.39841264486312866, |
| "reward_std": 0.5363882184028625, |
| "rewards/reward_func/mean": 0.39841264486312866, |
| "rewards/reward_func/std": 0.5363882184028625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.871351718902588, |
| "sampling/importance_sampling_ratio/mean": 0.9914729595184326, |
| "sampling/importance_sampling_ratio/min": 0.09377329051494598, |
| "sampling/sampling_logp_difference/max": 2.366875171661377, |
| "sampling/sampling_logp_difference/mean": 0.017338084056973457, |
| "step": 96, |
| "step_time": 435.25480223365594 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14781.0, |
| "completions/max_terminated_length": 14781.0, |
| "completions/mean_length": 10244.75, |
| "completions/mean_terminated_length": 10244.75, |
| "completions/min_length": 5004.0, |
| "completions/min_terminated_length": 5004.0, |
| "entropy": 0.3078562542796135, |
| "epoch": 0.11644657863145258, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4103779196739197, |
| "learning_rate": 6.16e-07, |
| "loss": 0.038, |
| "num_tokens": 34434571.0, |
| "reward": 0.8614925146102905, |
| "reward_std": 0.5891639590263367, |
| "rewards/reward_func/mean": 0.8614925146102905, |
| "rewards/reward_func/std": 0.5891639590263367, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2111990451812744, |
| "sampling/importance_sampling_ratio/mean": 0.9891133308410645, |
| "sampling/importance_sampling_ratio/min": 0.1545713096857071, |
| "sampling/sampling_logp_difference/max": 1.8670997619628906, |
| "sampling/sampling_logp_difference/mean": 0.021383928135037422, |
| "step": 97, |
| "step_time": 212.63684044289403 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15325.0, |
| "completions/max_terminated_length": 15325.0, |
| "completions/mean_length": 11469.84375, |
| "completions/mean_terminated_length": 11469.84375, |
| "completions/min_length": 6985.0, |
| "completions/min_terminated_length": 6985.0, |
| "entropy": 0.26262489426881075, |
| "epoch": 0.11764705882352941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37121981382369995, |
| "learning_rate": 6.119999999999999e-07, |
| "loss": 0.0254, |
| "num_tokens": 34846182.0, |
| "reward": 1.391387701034546, |
| "reward_std": 2.258481025695801, |
| "rewards/reward_func/mean": 1.391387701034546, |
| "rewards/reward_func/std": 2.258481025695801, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9727016687393188, |
| "sampling/importance_sampling_ratio/mean": 0.9908169507980347, |
| "sampling/importance_sampling_ratio/min": 0.24390530586242676, |
| "sampling/sampling_logp_difference/max": 1.4109752178192139, |
| "sampling/sampling_logp_difference/mean": 0.018506620079278946, |
| "step": 98, |
| "step_time": 251.3301967199659 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12299.0, |
| "completions/max_terminated_length": 12299.0, |
| "completions/mean_length": 8463.15625, |
| "completions/mean_terminated_length": 8463.15625, |
| "completions/min_length": 3478.0, |
| "completions/min_terminated_length": 3478.0, |
| "entropy": 0.273245669901371, |
| "epoch": 0.11884753901560624, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.365919291973114, |
| "learning_rate": 6.079999999999999e-07, |
| "loss": 0.0383, |
| "num_tokens": 35145083.0, |
| "reward": 3.128838539123535, |
| "reward_std": 3.8530807495117188, |
| "rewards/reward_func/mean": 3.128838539123535, |
| "rewards/reward_func/std": 3.853080987930298, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.477442979812622, |
| "sampling/importance_sampling_ratio/mean": 0.9905266761779785, |
| "sampling/importance_sampling_ratio/min": 0.38133692741394043, |
| "sampling/sampling_logp_difference/max": 0.9640719890594482, |
| "sampling/sampling_logp_difference/mean": 0.0193619467318058, |
| "step": 99, |
| "step_time": 214.26986349211074 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14368.0, |
| "completions/max_terminated_length": 14368.0, |
| "completions/mean_length": 8256.4375, |
| "completions/mean_terminated_length": 8256.4375, |
| "completions/min_length": 4534.0, |
| "completions/min_terminated_length": 4534.0, |
| "entropy": 0.3049591239541769, |
| "epoch": 0.12004801920768307, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.38011953234672546, |
| "learning_rate": 6.04e-07, |
| "loss": -0.0081, |
| "num_tokens": 35432593.0, |
| "reward": 2.3642165660858154, |
| "reward_std": 2.706695556640625, |
| "rewards/reward_func/mean": 2.3642165660858154, |
| "rewards/reward_func/std": 2.706695556640625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9068528413772583, |
| "sampling/importance_sampling_ratio/mean": 0.9893739223480225, |
| "sampling/importance_sampling_ratio/min": 0.1614706665277481, |
| "sampling/sampling_logp_difference/max": 1.8234318494796753, |
| "sampling/sampling_logp_difference/mean": 0.02130398340523243, |
| "step": 100, |
| "step_time": 184.41112436982803 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17413.0, |
| "completions/max_terminated_length": 17413.0, |
| "completions/mean_length": 11627.0625, |
| "completions/mean_terminated_length": 11627.0625, |
| "completions/min_length": 6505.0, |
| "completions/min_terminated_length": 6505.0, |
| "entropy": 0.24142704904079437, |
| "epoch": 0.1212484993997599, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36239883303642273, |
| "learning_rate": 6e-07, |
| "loss": 0.0495, |
| "num_tokens": 35847699.0, |
| "reward": 0.8143461346626282, |
| "reward_std": 0.6679160594940186, |
| "rewards/reward_func/mean": 0.8221967816352844, |
| "rewards/reward_func/std": 0.6563639044761658, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00785064697265625, |
| "rewards/soft_overlong_punishment_reward/std": 0.044409964233636856, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9916003346443176, |
| "sampling/importance_sampling_ratio/min": 0.05457422509789467, |
| "sampling/sampling_logp_difference/max": 2.908193588256836, |
| "sampling/sampling_logp_difference/mean": 0.01743597723543644, |
| "step": 101, |
| "step_time": 247.9607744079549 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12313.0, |
| "completions/max_terminated_length": 12313.0, |
| "completions/mean_length": 9030.0625, |
| "completions/mean_terminated_length": 9030.0625, |
| "completions/min_length": 6142.0, |
| "completions/min_terminated_length": 6142.0, |
| "entropy": 0.2866959013044834, |
| "epoch": 0.12244897959183673, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4205459654331207, |
| "learning_rate": 5.96e-07, |
| "loss": 0.0247, |
| "num_tokens": 36166773.0, |
| "reward": 1.333907127380371, |
| "reward_std": 1.1282483339309692, |
| "rewards/reward_func/mean": 1.333907127380371, |
| "rewards/reward_func/std": 1.1282483339309692, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.730395555496216, |
| "sampling/importance_sampling_ratio/mean": 0.9900311231613159, |
| "sampling/importance_sampling_ratio/min": 0.05878521502017975, |
| "sampling/sampling_logp_difference/max": 2.83386492729187, |
| "sampling/sampling_logp_difference/mean": 0.020093757659196854, |
| "step": 102, |
| "step_time": 215.10454945149831 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12475.0, |
| "completions/max_terminated_length": 12475.0, |
| "completions/mean_length": 8976.34375, |
| "completions/mean_terminated_length": 8976.34375, |
| "completions/min_length": 3906.0, |
| "completions/min_terminated_length": 3906.0, |
| "entropy": 0.24582395143806934, |
| "epoch": 0.12364945978391356, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3737013339996338, |
| "learning_rate": 5.919999999999999e-07, |
| "loss": 0.0012, |
| "num_tokens": 36486264.0, |
| "reward": 0.4444224536418915, |
| "reward_std": 0.6303268671035767, |
| "rewards/reward_func/mean": 0.4444224536418915, |
| "rewards/reward_func/std": 0.6303268074989319, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9914492964744568, |
| "sampling/importance_sampling_ratio/min": 0.18756607174873352, |
| "sampling/sampling_logp_difference/max": 1.673624038696289, |
| "sampling/sampling_logp_difference/mean": 0.01754648983478546, |
| "step": 103, |
| "step_time": 206.92123383819126 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13655.0, |
| "completions/max_terminated_length": 13655.0, |
| "completions/mean_length": 9150.15625, |
| "completions/mean_terminated_length": 9150.15625, |
| "completions/min_length": 4648.0, |
| "completions/min_terminated_length": 4648.0, |
| "entropy": 0.26344494987279177, |
| "epoch": 0.12484993997599039, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.34451353549957275, |
| "learning_rate": 5.879999999999999e-07, |
| "loss": -0.0295, |
| "num_tokens": 36814093.0, |
| "reward": 5.4827399253845215, |
| "reward_std": 10.697521209716797, |
| "rewards/reward_func/mean": 5.4827399253845215, |
| "rewards/reward_func/std": 10.697521209716797, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3717105388641357, |
| "sampling/importance_sampling_ratio/mean": 0.9908387064933777, |
| "sampling/importance_sampling_ratio/min": 0.23612551391124725, |
| "sampling/sampling_logp_difference/max": 1.4433917999267578, |
| "sampling/sampling_logp_difference/mean": 0.018703076988458633, |
| "step": 104, |
| "step_time": 213.1645632191794 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14144.0, |
| "completions/max_terminated_length": 14144.0, |
| "completions/mean_length": 9819.1875, |
| "completions/mean_terminated_length": 9819.1875, |
| "completions/min_length": 5625.0, |
| "completions/min_terminated_length": 5625.0, |
| "entropy": 0.24409340787678957, |
| "epoch": 0.12605042016806722, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.33012041449546814, |
| "learning_rate": 5.839999999999999e-07, |
| "loss": 0.029, |
| "num_tokens": 37167963.0, |
| "reward": 1.2769612073898315, |
| "reward_std": 2.2972137928009033, |
| "rewards/reward_func/mean": 1.2769612073898315, |
| "rewards/reward_func/std": 2.2972140312194824, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3744382858276367, |
| "sampling/importance_sampling_ratio/mean": 0.9916211366653442, |
| "sampling/importance_sampling_ratio/min": 0.18144047260284424, |
| "sampling/sampling_logp_difference/max": 1.7068276405334473, |
| "sampling/sampling_logp_difference/mean": 0.017610425129532814, |
| "step": 105, |
| "step_time": 233.00164715701248 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19045.0, |
| "completions/max_terminated_length": 19045.0, |
| "completions/mean_length": 10475.28125, |
| "completions/mean_terminated_length": 10475.28125, |
| "completions/min_length": 6236.0, |
| "completions/min_terminated_length": 6236.0, |
| "entropy": 0.2810442354530096, |
| "epoch": 0.12725090036014405, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4047479033470154, |
| "learning_rate": 5.8e-07, |
| "loss": 0.0844, |
| "num_tokens": 37540988.0, |
| "reward": 1.971191644668579, |
| "reward_std": 3.9106719493865967, |
| "rewards/reward_func/mean": 1.9987413883209229, |
| "rewards/reward_func/std": 3.894240140914917, |
| "rewards/soft_overlong_punishment_reward/mean": -0.02754974365234375, |
| "rewards/soft_overlong_punishment_reward/std": 0.12069173902273178, |
| "sampling/importance_sampling_ratio/max": 2.3556199073791504, |
| "sampling/importance_sampling_ratio/mean": 0.990192711353302, |
| "sampling/importance_sampling_ratio/min": 0.1058955043554306, |
| "sampling/sampling_logp_difference/max": 2.245302438735962, |
| "sampling/sampling_logp_difference/mean": 0.019729377701878548, |
| "step": 106, |
| "step_time": 259.3078554049134 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11035.0, |
| "completions/max_terminated_length": 11035.0, |
| "completions/mean_length": 8085.09375, |
| "completions/mean_terminated_length": 8085.09375, |
| "completions/min_length": 5843.0, |
| "completions/min_terminated_length": 5843.0, |
| "entropy": 0.28292676340788603, |
| "epoch": 0.12845138055222088, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.44497084617614746, |
| "learning_rate": 5.76e-07, |
| "loss": -0.0027, |
| "num_tokens": 37824127.0, |
| "reward": 5.807962417602539, |
| "reward_std": 3.6478431224823, |
| "rewards/reward_func/mean": 5.807962417602539, |
| "rewards/reward_func/std": 3.6478431224823, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9902195930480957, |
| "sampling/importance_sampling_ratio/min": 0.2602680027484894, |
| "sampling/sampling_logp_difference/max": 1.6943206787109375, |
| "sampling/sampling_logp_difference/mean": 0.01987529546022415, |
| "step": 107, |
| "step_time": 164.33568983571604 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16656.0, |
| "completions/max_terminated_length": 16656.0, |
| "completions/mean_length": 10496.1875, |
| "completions/mean_terminated_length": 10496.1875, |
| "completions/min_length": 6152.0, |
| "completions/min_terminated_length": 6152.0, |
| "entropy": 0.31788551434874535, |
| "epoch": 0.12965186074429771, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36823001503944397, |
| "learning_rate": 5.719999999999999e-07, |
| "loss": 0.0434, |
| "num_tokens": 38184413.0, |
| "reward": 0.822684109210968, |
| "reward_std": 0.7153370380401611, |
| "rewards/reward_func/mean": 0.824759304523468, |
| "rewards/reward_func/std": 0.7129164934158325, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0020751953125, |
| "rewards/soft_overlong_punishment_reward/std": 0.011739077046513557, |
| "sampling/importance_sampling_ratio/max": 2.7742605209350586, |
| "sampling/importance_sampling_ratio/mean": 0.9888496994972229, |
| "sampling/importance_sampling_ratio/min": 0.3614434599876404, |
| "sampling/sampling_logp_difference/max": 1.0203841924667358, |
| "sampling/sampling_logp_difference/mean": 0.0220322422683239, |
| "step": 108, |
| "step_time": 238.20601075387094 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14831.0, |
| "completions/max_terminated_length": 14831.0, |
| "completions/mean_length": 10238.46875, |
| "completions/mean_terminated_length": 10238.46875, |
| "completions/min_length": 6359.0, |
| "completions/min_terminated_length": 6359.0, |
| "entropy": 0.28053270652890205, |
| "epoch": 0.13085234093637454, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39827480912208557, |
| "learning_rate": 5.679999999999999e-07, |
| "loss": -0.0661, |
| "num_tokens": 38553132.0, |
| "reward": 0.5400049686431885, |
| "reward_std": 0.8026726245880127, |
| "rewards/reward_func/mean": 0.5400049686431885, |
| "rewards/reward_func/std": 0.8026726841926575, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3182525634765625, |
| "sampling/importance_sampling_ratio/mean": 0.9902611374855042, |
| "sampling/importance_sampling_ratio/min": 0.3386656939983368, |
| "sampling/sampling_logp_difference/max": 1.0827418565750122, |
| "sampling/sampling_logp_difference/mean": 0.01999467983841896, |
| "step": 109, |
| "step_time": 251.36417699581943 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12513.0, |
| "completions/max_terminated_length": 12513.0, |
| "completions/mean_length": 9081.59375, |
| "completions/mean_terminated_length": 9081.59375, |
| "completions/min_length": 4968.0, |
| "completions/min_terminated_length": 4968.0, |
| "entropy": 0.2736341943964362, |
| "epoch": 0.13205282112845138, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4176849126815796, |
| "learning_rate": 5.639999999999999e-07, |
| "loss": 0.0632, |
| "num_tokens": 38883943.0, |
| "reward": 1.397516131401062, |
| "reward_std": 1.4397801160812378, |
| "rewards/reward_func/mean": 1.397516131401062, |
| "rewards/reward_func/std": 1.4397801160812378, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.864140510559082, |
| "sampling/importance_sampling_ratio/mean": 0.9903908371925354, |
| "sampling/importance_sampling_ratio/min": 0.30289867520332336, |
| "sampling/sampling_logp_difference/max": 1.194356918334961, |
| "sampling/sampling_logp_difference/mean": 0.019294166937470436, |
| "step": 110, |
| "step_time": 432.76785584609024 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15302.0, |
| "completions/max_terminated_length": 15302.0, |
| "completions/mean_length": 10490.46875, |
| "completions/mean_terminated_length": 10490.46875, |
| "completions/min_length": 7700.0, |
| "completions/min_terminated_length": 7700.0, |
| "entropy": 0.2960231024771929, |
| "epoch": 0.1332533013205282, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4143314063549042, |
| "learning_rate": 5.6e-07, |
| "loss": 0.0137, |
| "num_tokens": 39249678.0, |
| "reward": 1.0926156044006348, |
| "reward_std": 0.6936445832252502, |
| "rewards/reward_func/mean": 1.0926156044006348, |
| "rewards/reward_func/std": 0.6936445236206055, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.972447633743286, |
| "sampling/importance_sampling_ratio/mean": 0.9898686408996582, |
| "sampling/importance_sampling_ratio/min": 0.15995651483535767, |
| "sampling/sampling_logp_difference/max": 1.8328533172607422, |
| "sampling/sampling_logp_difference/mean": 0.02054767683148384, |
| "step": 111, |
| "step_time": 220.7248961061705 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17000.0, |
| "completions/max_terminated_length": 17000.0, |
| "completions/mean_length": 9145.03125, |
| "completions/mean_terminated_length": 9145.03125, |
| "completions/min_length": 3641.0, |
| "completions/min_terminated_length": 3641.0, |
| "entropy": 0.2596113607287407, |
| "epoch": 0.13445378151260504, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42476722598075867, |
| "learning_rate": 5.560000000000001e-07, |
| "loss": 0.0171, |
| "num_tokens": 39581495.0, |
| "reward": 0.21374335885047913, |
| "reward_std": 0.40230268239974976, |
| "rewards/reward_func/mean": 0.21844306588172913, |
| "rewards/reward_func/std": 0.39877456426620483, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00469970703125, |
| "rewards/soft_overlong_punishment_reward/std": 0.02658555842936039, |
| "sampling/importance_sampling_ratio/max": 2.445462942123413, |
| "sampling/importance_sampling_ratio/mean": 0.9910917282104492, |
| "sampling/importance_sampling_ratio/min": 0.0019266947638243437, |
| "sampling/sampling_logp_difference/max": 6.251949310302734, |
| "sampling/sampling_logp_difference/mean": 0.018219787627458572, |
| "step": 112, |
| "step_time": 280.31022595707327 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13890.0, |
| "completions/max_terminated_length": 13890.0, |
| "completions/mean_length": 10680.34375, |
| "completions/mean_terminated_length": 10680.34375, |
| "completions/min_length": 6448.0, |
| "completions/min_terminated_length": 6448.0, |
| "entropy": 0.28790590912103653, |
| "epoch": 0.13565426170468187, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3604806065559387, |
| "learning_rate": 5.520000000000001e-07, |
| "loss": -0.0006, |
| "num_tokens": 39955530.0, |
| "reward": 0.5530512928962708, |
| "reward_std": 0.9826109409332275, |
| "rewards/reward_func/mean": 0.5530512928962708, |
| "rewards/reward_func/std": 0.9826108813285828, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.565842866897583, |
| "sampling/importance_sampling_ratio/mean": 0.9898175597190857, |
| "sampling/importance_sampling_ratio/min": 0.2070624679327011, |
| "sampling/sampling_logp_difference/max": 1.5747346878051758, |
| "sampling/sampling_logp_difference/mean": 0.02044212445616722, |
| "step": 113, |
| "step_time": 420.17655016179197 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13642.0, |
| "completions/max_terminated_length": 13642.0, |
| "completions/mean_length": 9351.1875, |
| "completions/mean_terminated_length": 9351.1875, |
| "completions/min_length": 5824.0, |
| "completions/min_terminated_length": 5824.0, |
| "entropy": 0.2932744771242142, |
| "epoch": 0.1368547418967587, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4188636839389801, |
| "learning_rate": 5.48e-07, |
| "loss": 0.004, |
| "num_tokens": 40283608.0, |
| "reward": 1.3525290489196777, |
| "reward_std": 1.7022446393966675, |
| "rewards/reward_func/mean": 1.3525290489196777, |
| "rewards/reward_func/std": 1.702244520187378, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989858090877533, |
| "sampling/importance_sampling_ratio/min": 0.2756859362125397, |
| "sampling/sampling_logp_difference/max": 2.190220355987549, |
| "sampling/sampling_logp_difference/mean": 0.020323438569903374, |
| "step": 114, |
| "step_time": 226.98730401217472 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17342.0, |
| "completions/max_terminated_length": 17342.0, |
| "completions/mean_length": 8931.0, |
| "completions/mean_terminated_length": 8931.0, |
| "completions/min_length": 3315.0, |
| "completions/min_terminated_length": 3315.0, |
| "entropy": 0.24458013847470284, |
| "epoch": 0.13805522208883553, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3886120617389679, |
| "learning_rate": 5.44e-07, |
| "loss": -0.0341, |
| "num_tokens": 40610928.0, |
| "reward": 0.9550597667694092, |
| "reward_std": 1.4101365804672241, |
| "rewards/reward_func/mean": 0.9623687267303467, |
| "rewards/reward_func/std": 1.4116019010543823, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0073089599609375, |
| "rewards/soft_overlong_punishment_reward/std": 0.04134571924805641, |
| "sampling/importance_sampling_ratio/max": 2.6611251831054688, |
| "sampling/importance_sampling_ratio/mean": 0.9916718006134033, |
| "sampling/importance_sampling_ratio/min": 0.16927804052829742, |
| "sampling/sampling_logp_difference/max": 1.7762126922607422, |
| "sampling/sampling_logp_difference/mean": 0.017050782218575478, |
| "step": 115, |
| "step_time": 229.30203108501155 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15122.0, |
| "completions/max_terminated_length": 15122.0, |
| "completions/mean_length": 10591.75, |
| "completions/mean_terminated_length": 10591.75, |
| "completions/min_length": 7260.0, |
| "completions/min_terminated_length": 7260.0, |
| "entropy": 0.2582763684913516, |
| "epoch": 0.13925570228091236, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3612559139728546, |
| "learning_rate": 5.4e-07, |
| "loss": 0.0453, |
| "num_tokens": 40996336.0, |
| "reward": 2.7374861240386963, |
| "reward_std": 3.0827701091766357, |
| "rewards/reward_func/mean": 2.7374861240386963, |
| "rewards/reward_func/std": 3.0827698707580566, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.93190598487854, |
| "sampling/importance_sampling_ratio/mean": 0.9909803867340088, |
| "sampling/importance_sampling_ratio/min": 0.031009899452328682, |
| "sampling/sampling_logp_difference/max": 3.4734487533569336, |
| "sampling/sampling_logp_difference/mean": 0.018270010128617287, |
| "step": 116, |
| "step_time": 227.7563733382849 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13868.0, |
| "completions/max_terminated_length": 13868.0, |
| "completions/mean_length": 9498.53125, |
| "completions/mean_terminated_length": 9498.53125, |
| "completions/min_length": 5313.0, |
| "completions/min_terminated_length": 5313.0, |
| "entropy": 0.2784241568297148, |
| "epoch": 0.1404561824729892, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5930799841880798, |
| "learning_rate": 5.36e-07, |
| "loss": -0.033, |
| "num_tokens": 41336345.0, |
| "reward": 0.9384326338768005, |
| "reward_std": 1.186398983001709, |
| "rewards/reward_func/mean": 0.9384326338768005, |
| "rewards/reward_func/std": 1.1863987445831299, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.455018997192383, |
| "sampling/importance_sampling_ratio/mean": 0.9903002977371216, |
| "sampling/importance_sampling_ratio/min": 0.015267138369381428, |
| "sampling/sampling_logp_difference/max": 4.1820526123046875, |
| "sampling/sampling_logp_difference/mean": 0.019709795713424683, |
| "step": 117, |
| "step_time": 243.17280280101113 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14484.0, |
| "completions/max_terminated_length": 14484.0, |
| "completions/mean_length": 9971.8125, |
| "completions/mean_terminated_length": 9971.8125, |
| "completions/min_length": 7100.0, |
| "completions/min_terminated_length": 7100.0, |
| "entropy": 0.29973538406193256, |
| "epoch": 0.14165666266506602, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5883257985115051, |
| "learning_rate": 5.32e-07, |
| "loss": 0.0281, |
| "num_tokens": 41688307.0, |
| "reward": 0.6539914608001709, |
| "reward_std": 0.7319720983505249, |
| "rewards/reward_func/mean": 0.6539914608001709, |
| "rewards/reward_func/std": 0.7319720387458801, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.818315029144287, |
| "sampling/importance_sampling_ratio/mean": 0.9895184636116028, |
| "sampling/importance_sampling_ratio/min": 0.4023731052875519, |
| "sampling/sampling_logp_difference/max": 1.0361392498016357, |
| "sampling/sampling_logp_difference/mean": 0.021014517173171043, |
| "step": 118, |
| "step_time": 231.1251015268499 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19530.0, |
| "completions/max_terminated_length": 19530.0, |
| "completions/mean_length": 11520.625, |
| "completions/mean_terminated_length": 11520.625, |
| "completions/min_length": 3825.0, |
| "completions/min_terminated_length": 3825.0, |
| "entropy": 0.24881694186478853, |
| "epoch": 0.14285714285714285, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3857250213623047, |
| "learning_rate": 5.28e-07, |
| "loss": -0.0142, |
| "num_tokens": 42102007.0, |
| "reward": 1.7721220254898071, |
| "reward_std": 2.1013307571411133, |
| "rewards/reward_func/mean": 1.8261228799819946, |
| "rewards/reward_func/std": 2.050947427749634, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0540008544921875, |
| "rewards/soft_overlong_punishment_reward/std": 0.18218861520290375, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9912911653518677, |
| "sampling/importance_sampling_ratio/min": 0.07585831731557846, |
| "sampling/sampling_logp_difference/max": 2.578887939453125, |
| "sampling/sampling_logp_difference/mean": 0.017922591418027878, |
| "step": 119, |
| "step_time": 291.4575341643067 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15462.0, |
| "completions/max_terminated_length": 15462.0, |
| "completions/mean_length": 8598.9375, |
| "completions/mean_terminated_length": 8598.9375, |
| "completions/min_length": 3214.0, |
| "completions/min_terminated_length": 3214.0, |
| "entropy": 0.25942704640328884, |
| "epoch": 0.14405762304921968, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.37554553151130676, |
| "learning_rate": 5.24e-07, |
| "loss": 0.0008, |
| "num_tokens": 42406077.0, |
| "reward": 1.3100762367248535, |
| "reward_std": 1.914520263671875, |
| "rewards/reward_func/mean": 1.3100762367248535, |
| "rewards/reward_func/std": 1.914520263671875, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4972586631774902, |
| "sampling/importance_sampling_ratio/mean": 0.9908623099327087, |
| "sampling/importance_sampling_ratio/min": 0.1831483691930771, |
| "sampling/sampling_logp_difference/max": 1.6974586248397827, |
| "sampling/sampling_logp_difference/mean": 0.01822926476597786, |
| "step": 120, |
| "step_time": 209.00005609192885 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16215.0, |
| "completions/max_terminated_length": 16215.0, |
| "completions/mean_length": 9589.25, |
| "completions/mean_terminated_length": 9589.25, |
| "completions/min_length": 5802.0, |
| "completions/min_terminated_length": 5802.0, |
| "entropy": 0.3083428144454956, |
| "epoch": 0.1452581032412965, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4060857594013214, |
| "learning_rate": 5.2e-07, |
| "loss": -0.0524, |
| "num_tokens": 42737701.0, |
| "reward": 0.49280887842178345, |
| "reward_std": 0.6724991798400879, |
| "rewards/reward_func/mean": 0.49280887842178345, |
| "rewards/reward_func/std": 0.6724991798400879, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.6253695487976074, |
| "sampling/importance_sampling_ratio/mean": 0.9890679717063904, |
| "sampling/importance_sampling_ratio/min": 0.00019750210049096495, |
| "sampling/sampling_logp_difference/max": 8.52976131439209, |
| "sampling/sampling_logp_difference/mean": 0.021406318992376328, |
| "step": 121, |
| "step_time": 241.5949507113546 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15308.0, |
| "completions/max_terminated_length": 15308.0, |
| "completions/mean_length": 10594.34375, |
| "completions/mean_terminated_length": 10594.34375, |
| "completions/min_length": 7324.0, |
| "completions/min_terminated_length": 7324.0, |
| "entropy": 0.28266207315027714, |
| "epoch": 0.14645858343337334, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.2750304937362671, |
| "learning_rate": 5.16e-07, |
| "loss": 0.0195, |
| "num_tokens": 43113840.0, |
| "reward": 0.5235170722007751, |
| "reward_std": 0.633671224117279, |
| "rewards/reward_func/mean": 0.5235170722007751, |
| "rewards/reward_func/std": 0.633671224117279, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4920129776000977, |
| "sampling/importance_sampling_ratio/mean": 0.9901369214057922, |
| "sampling/importance_sampling_ratio/min": 0.10062485188245773, |
| "sampling/sampling_logp_difference/max": 2.296355962753296, |
| "sampling/sampling_logp_difference/mean": 0.01979176700115204, |
| "step": 122, |
| "step_time": 225.33721455116756 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15019.0, |
| "completions/max_terminated_length": 15019.0, |
| "completions/mean_length": 11032.53125, |
| "completions/mean_terminated_length": 11032.53125, |
| "completions/min_length": 5304.0, |
| "completions/min_terminated_length": 5304.0, |
| "entropy": 0.26705341041088104, |
| "epoch": 0.14765906362545017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3584786653518677, |
| "learning_rate": 5.12e-07, |
| "loss": 0.0368, |
| "num_tokens": 43507401.0, |
| "reward": 1.9813694953918457, |
| "reward_std": 2.857267141342163, |
| "rewards/reward_func/mean": 1.9813694953918457, |
| "rewards/reward_func/std": 2.857267379760742, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2451746463775635, |
| "sampling/importance_sampling_ratio/mean": 0.9905773997306824, |
| "sampling/importance_sampling_ratio/min": 0.07104846835136414, |
| "sampling/sampling_logp_difference/max": 2.644392967224121, |
| "sampling/sampling_logp_difference/mean": 0.019043972715735435, |
| "step": 123, |
| "step_time": 460.36738448997494 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 20032.0, |
| "completions/max_terminated_length": 20032.0, |
| "completions/mean_length": 11611.53125, |
| "completions/mean_terminated_length": 11611.53125, |
| "completions/min_length": 6363.0, |
| "completions/min_terminated_length": 6363.0, |
| "entropy": 0.2338911509141326, |
| "epoch": 0.148859543817527, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3491296172142029, |
| "learning_rate": 5.079999999999999e-07, |
| "loss": 0.0364, |
| "num_tokens": 43926578.0, |
| "reward": 1.0934959650039673, |
| "reward_std": 1.0090538263320923, |
| "rewards/reward_func/mean": 1.1581779718399048, |
| "rewards/reward_func/std": 0.9206379055976868, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0646820068359375, |
| "rewards/soft_overlong_punishment_reward/std": 0.220370814204216, |
| "sampling/importance_sampling_ratio/max": 1.99778151512146, |
| "sampling/importance_sampling_ratio/mean": 0.9919837713241577, |
| "sampling/importance_sampling_ratio/min": 0.01377395261079073, |
| "sampling/sampling_logp_difference/max": 4.284976005554199, |
| "sampling/sampling_logp_difference/mean": 0.016360996291041374, |
| "step": 124, |
| "step_time": 275.2218638029881 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16157.0, |
| "completions/max_terminated_length": 16157.0, |
| "completions/mean_length": 10606.0625, |
| "completions/mean_terminated_length": 10606.0625, |
| "completions/min_length": 4477.0, |
| "completions/min_terminated_length": 4477.0, |
| "entropy": 0.2876010574400425, |
| "epoch": 0.15006002400960383, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.4879244565963745, |
| "learning_rate": 5.04e-07, |
| "loss": 0.0299, |
| "num_tokens": 44300284.0, |
| "reward": 8.492034912109375, |
| "reward_std": 24.353479385375977, |
| "rewards/reward_func/mean": 8.492034912109375, |
| "rewards/reward_func/std": 24.35348129272461, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.82053542137146, |
| "sampling/importance_sampling_ratio/mean": 0.9900239109992981, |
| "sampling/importance_sampling_ratio/min": 0.2887732684612274, |
| "sampling/sampling_logp_difference/max": 1.242113471031189, |
| "sampling/sampling_logp_difference/mean": 0.020131349563598633, |
| "step": 125, |
| "step_time": 262.3409889435861 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15917.0, |
| "completions/max_terminated_length": 15917.0, |
| "completions/mean_length": 9516.4375, |
| "completions/mean_terminated_length": 9516.4375, |
| "completions/min_length": 4847.0, |
| "completions/min_terminated_length": 4847.0, |
| "entropy": 0.2908124439418316, |
| "epoch": 0.15126050420168066, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38954854011535645, |
| "learning_rate": 5e-07, |
| "loss": 0.0245, |
| "num_tokens": 44634890.0, |
| "reward": 1.9146395921707153, |
| "reward_std": 2.8896992206573486, |
| "rewards/reward_func/mean": 1.9146395921707153, |
| "rewards/reward_func/std": 2.8896987438201904, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4463016986846924, |
| "sampling/importance_sampling_ratio/mean": 0.9897171854972839, |
| "sampling/importance_sampling_ratio/min": 0.3122645914554596, |
| "sampling/sampling_logp_difference/max": 1.1639044284820557, |
| "sampling/sampling_logp_difference/mean": 0.020406031981110573, |
| "step": 126, |
| "step_time": 262.93810594966635 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15624.0, |
| "completions/max_terminated_length": 15624.0, |
| "completions/mean_length": 9908.875, |
| "completions/mean_terminated_length": 9908.875, |
| "completions/min_length": 6398.0, |
| "completions/min_terminated_length": 6398.0, |
| "entropy": 0.3054868131875992, |
| "epoch": 0.1524609843937575, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3650784492492676, |
| "learning_rate": 4.96e-07, |
| "loss": 0.0082, |
| "num_tokens": 44975030.0, |
| "reward": 1.5860934257507324, |
| "reward_std": 2.869269847869873, |
| "rewards/reward_func/mean": 1.5860934257507324, |
| "rewards/reward_func/std": 2.869270086288452, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.127547025680542, |
| "sampling/importance_sampling_ratio/mean": 0.9892829060554504, |
| "sampling/importance_sampling_ratio/min": 0.21681442856788635, |
| "sampling/sampling_logp_difference/max": 1.5287134647369385, |
| "sampling/sampling_logp_difference/mean": 0.02112971805036068, |
| "step": 127, |
| "step_time": 216.3248087487882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14126.0, |
| "completions/max_terminated_length": 14126.0, |
| "completions/mean_length": 7167.53125, |
| "completions/mean_terminated_length": 7167.53125, |
| "completions/min_length": 5186.0, |
| "completions/min_terminated_length": 5186.0, |
| "entropy": 0.2729322211816907, |
| "epoch": 0.15366146458583432, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.44890305399894714, |
| "learning_rate": 4.92e-07, |
| "loss": 0.0657, |
| "num_tokens": 45233063.0, |
| "reward": 3.381237745285034, |
| "reward_std": 3.5361828804016113, |
| "rewards/reward_func/mean": 3.381237745285034, |
| "rewards/reward_func/std": 3.5361826419830322, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.002379894256592, |
| "sampling/importance_sampling_ratio/mean": 0.9904590845108032, |
| "sampling/importance_sampling_ratio/min": 0.3677864670753479, |
| "sampling/sampling_logp_difference/max": 1.0002528429031372, |
| "sampling/sampling_logp_difference/mean": 0.019061625003814697, |
| "step": 128, |
| "step_time": 196.9817350585945 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15338.0, |
| "completions/max_terminated_length": 15338.0, |
| "completions/mean_length": 7408.9375, |
| "completions/mean_terminated_length": 7408.9375, |
| "completions/min_length": 2486.0, |
| "completions/min_terminated_length": 2486.0, |
| "entropy": 0.28559087216854095, |
| "epoch": 0.15486194477791115, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4474518895149231, |
| "learning_rate": 4.879999999999999e-07, |
| "loss": 0.0143, |
| "num_tokens": 45501781.0, |
| "reward": 1.7653834819793701, |
| "reward_std": 1.865670919418335, |
| "rewards/reward_func/mean": 1.7653834819793701, |
| "rewards/reward_func/std": 1.865670919418335, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.251772165298462, |
| "sampling/importance_sampling_ratio/mean": 0.9898481369018555, |
| "sampling/importance_sampling_ratio/min": 0.33571237325668335, |
| "sampling/sampling_logp_difference/max": 1.0915005207061768, |
| "sampling/sampling_logp_difference/mean": 0.020369449630379677, |
| "step": 129, |
| "step_time": 213.23672278015874 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.09375, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 13311.0, |
| "completions/mean_length": 10419.03125, |
| "completions/mean_terminated_length": 9378.2412109375, |
| "completions/min_length": 6891.0, |
| "completions/min_terminated_length": 6891.0, |
| "entropy": 0.288710068911314, |
| "epoch": 0.15606242496998798, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37506240606307983, |
| "learning_rate": 4.839999999999999e-07, |
| "loss": -0.1003, |
| "num_tokens": 45864014.0, |
| "reward": 1.1006951332092285, |
| "reward_std": 1.99079167842865, |
| "rewards/reward_func/mean": 1.3180084228515625, |
| "rewards/reward_func/std": 1.9662554264068604, |
| "rewards/soft_overlong_punishment_reward/mean": -0.09375, |
| "rewards/soft_overlong_punishment_reward/std": 0.2961445748806, |
| "sampling/importance_sampling_ratio/max": 2.094200849533081, |
| "sampling/importance_sampling_ratio/mean": 0.9896755218505859, |
| "sampling/importance_sampling_ratio/min": 3.3655967854429036e-05, |
| "sampling/sampling_logp_difference/max": 10.299320220947266, |
| "sampling/sampling_logp_difference/mean": 0.02032674476504326, |
| "step": 130, |
| "step_time": 249.37841368792579 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13651.0, |
| "completions/max_terminated_length": 13651.0, |
| "completions/mean_length": 10875.9375, |
| "completions/mean_terminated_length": 10875.9375, |
| "completions/min_length": 6579.0, |
| "completions/min_terminated_length": 6579.0, |
| "entropy": 0.27954767271876335, |
| "epoch": 0.15726290516206481, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4141541123390198, |
| "learning_rate": 4.8e-07, |
| "loss": -0.0105, |
| "num_tokens": 46248332.0, |
| "reward": 0.5992684364318848, |
| "reward_std": 0.9581432342529297, |
| "rewards/reward_func/mean": 0.5992684364318848, |
| "rewards/reward_func/std": 0.9581431746482849, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2382140159606934, |
| "sampling/importance_sampling_ratio/mean": 0.990227997303009, |
| "sampling/importance_sampling_ratio/min": 0.3830912113189697, |
| "sampling/sampling_logp_difference/max": 0.9594821929931641, |
| "sampling/sampling_logp_difference/mean": 0.019494500011205673, |
| "step": 131, |
| "step_time": 450.12081778328866 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11920.0, |
| "completions/max_terminated_length": 11920.0, |
| "completions/mean_length": 7238.625, |
| "completions/mean_terminated_length": 7238.625, |
| "completions/min_length": 3557.0, |
| "completions/min_terminated_length": 3557.0, |
| "entropy": 0.2874604295939207, |
| "epoch": 0.15846338535414164, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5339971780776978, |
| "learning_rate": 4.76e-07, |
| "loss": 0.0164, |
| "num_tokens": 46507880.0, |
| "reward": 1.9809516668319702, |
| "reward_std": 1.8525370359420776, |
| "rewards/reward_func/mean": 1.9809516668319702, |
| "rewards/reward_func/std": 1.8525370359420776, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.424567699432373, |
| "sampling/importance_sampling_ratio/mean": 0.9898762702941895, |
| "sampling/importance_sampling_ratio/min": 0.3163217008113861, |
| "sampling/sampling_logp_difference/max": 1.1509954929351807, |
| "sampling/sampling_logp_difference/mean": 0.020033420994877815, |
| "step": 132, |
| "step_time": 163.30515668692533 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10824.0, |
| "completions/max_terminated_length": 10824.0, |
| "completions/mean_length": 8501.71875, |
| "completions/mean_terminated_length": 8501.71875, |
| "completions/min_length": 5532.0, |
| "completions/min_terminated_length": 5532.0, |
| "entropy": 0.2722770730033517, |
| "epoch": 0.15966386554621848, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4209204614162445, |
| "learning_rate": 4.7199999999999994e-07, |
| "loss": 0.0654, |
| "num_tokens": 46813647.0, |
| "reward": 5.072177886962891, |
| "reward_std": 4.542591094970703, |
| "rewards/reward_func/mean": 5.072177886962891, |
| "rewards/reward_func/std": 4.542591094970703, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.7433615922927856, |
| "sampling/importance_sampling_ratio/mean": 0.9904061555862427, |
| "sampling/importance_sampling_ratio/min": 0.09293912351131439, |
| "sampling/sampling_logp_difference/max": 2.3758106231689453, |
| "sampling/sampling_logp_difference/mean": 0.01888871379196644, |
| "step": 133, |
| "step_time": 183.62756524700671 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16072.0, |
| "completions/max_terminated_length": 16072.0, |
| "completions/mean_length": 11215.71875, |
| "completions/mean_terminated_length": 11215.71875, |
| "completions/min_length": 7240.0, |
| "completions/min_terminated_length": 7240.0, |
| "entropy": 0.22843076288700104, |
| "epoch": 0.1608643457382953, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3479200005531311, |
| "learning_rate": 4.68e-07, |
| "loss": -0.0415, |
| "num_tokens": 47226070.0, |
| "reward": 0.25698286294937134, |
| "reward_std": 0.8653408288955688, |
| "rewards/reward_func/mean": 0.25698286294937134, |
| "rewards/reward_func/std": 0.8653408288955688, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9921608567237854, |
| "sampling/importance_sampling_ratio/min": 8.29739210672642e-09, |
| "sampling/sampling_logp_difference/max": 18.607324600219727, |
| "sampling/sampling_logp_difference/mean": 0.0165218785405159, |
| "step": 134, |
| "step_time": 278.68348298675846 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15346.0, |
| "completions/max_terminated_length": 15346.0, |
| "completions/mean_length": 9651.1875, |
| "completions/mean_terminated_length": 9651.1875, |
| "completions/min_length": 4686.0, |
| "completions/min_terminated_length": 4686.0, |
| "entropy": 0.2711868965998292, |
| "epoch": 0.16206482593037214, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3969678580760956, |
| "learning_rate": 4.64e-07, |
| "loss": -0.0081, |
| "num_tokens": 47570140.0, |
| "reward": 2.622737407684326, |
| "reward_std": 3.003697633743286, |
| "rewards/reward_func/mean": 2.622737407684326, |
| "rewards/reward_func/std": 3.003697395324707, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.7643277645111084, |
| "sampling/importance_sampling_ratio/mean": 0.990485668182373, |
| "sampling/importance_sampling_ratio/min": 0.2230183184146881, |
| "sampling/sampling_logp_difference/max": 1.5005013942718506, |
| "sampling/sampling_logp_difference/mean": 0.018908120691776276, |
| "step": 135, |
| "step_time": 213.85006742121186 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11030.0, |
| "completions/max_terminated_length": 11030.0, |
| "completions/mean_length": 6975.90625, |
| "completions/mean_terminated_length": 6975.90625, |
| "completions/min_length": 3032.0, |
| "completions/min_terminated_length": 3032.0, |
| "entropy": 0.307786175981164, |
| "epoch": 0.16326530612244897, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.47706398367881775, |
| "learning_rate": 4.6e-07, |
| "loss": 0.0439, |
| "num_tokens": 47817641.0, |
| "reward": 2.0658512115478516, |
| "reward_std": 1.9194847345352173, |
| "rewards/reward_func/mean": 2.0658512115478516, |
| "rewards/reward_func/std": 1.9194847345352173, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.136054754257202, |
| "sampling/importance_sampling_ratio/mean": 0.9892805218696594, |
| "sampling/importance_sampling_ratio/min": 1.11404536667876e-15, |
| "sampling/sampling_logp_difference/max": 34.43077850341797, |
| "sampling/sampling_logp_difference/mean": 0.021540788933634758, |
| "step": 136, |
| "step_time": 174.63315520831384 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15303.0, |
| "completions/max_terminated_length": 15303.0, |
| "completions/mean_length": 10812.3125, |
| "completions/mean_terminated_length": 10812.3125, |
| "completions/min_length": 7491.0, |
| "completions/min_terminated_length": 7491.0, |
| "entropy": 0.2458400446921587, |
| "epoch": 0.1644657863145258, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.35999420285224915, |
| "learning_rate": 4.56e-07, |
| "loss": 0.0237, |
| "num_tokens": 48214651.0, |
| "reward": 2.3839621543884277, |
| "reward_std": 3.7139806747436523, |
| "rewards/reward_func/mean": 2.3839621543884277, |
| "rewards/reward_func/std": 3.7139809131622314, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1656391620635986, |
| "sampling/importance_sampling_ratio/mean": 0.9914739727973938, |
| "sampling/importance_sampling_ratio/min": 0.28416091203689575, |
| "sampling/sampling_logp_difference/max": 1.2582145929336548, |
| "sampling/sampling_logp_difference/mean": 0.017416341230273247, |
| "step": 137, |
| "step_time": 483.0681917546317 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15823.0, |
| "completions/max_terminated_length": 15823.0, |
| "completions/mean_length": 9706.625, |
| "completions/mean_terminated_length": 9706.625, |
| "completions/min_length": 3619.0, |
| "completions/min_terminated_length": 3619.0, |
| "entropy": 0.2981583010405302, |
| "epoch": 0.16566626650660263, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3441935181617737, |
| "learning_rate": 4.5199999999999997e-07, |
| "loss": 0.0477, |
| "num_tokens": 48558839.0, |
| "reward": 1027.4560546875, |
| "reward_std": 3014.18603515625, |
| "rewards/reward_func/mean": 1027.4560546875, |
| "rewards/reward_func/std": 3014.18603515625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989443838596344, |
| "sampling/importance_sampling_ratio/min": 0.0003842052537947893, |
| "sampling/sampling_logp_difference/max": 7.864333629608154, |
| "sampling/sampling_logp_difference/mean": 0.020724810659885406, |
| "step": 138, |
| "step_time": 429.2948167860741 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11754.0, |
| "completions/max_terminated_length": 11754.0, |
| "completions/mean_length": 9398.84375, |
| "completions/mean_terminated_length": 9398.84375, |
| "completions/min_length": 6889.0, |
| "completions/min_terminated_length": 6889.0, |
| "entropy": 0.24972048588097095, |
| "epoch": 0.16686674669867949, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5226235389709473, |
| "learning_rate": 4.48e-07, |
| "loss": 0.0647, |
| "num_tokens": 48899354.0, |
| "reward": 1.3123189210891724, |
| "reward_std": 1.0608717203140259, |
| "rewards/reward_func/mean": 1.3123189210891724, |
| "rewards/reward_func/std": 1.0608716011047363, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4963085651397705, |
| "sampling/importance_sampling_ratio/mean": 0.9913288354873657, |
| "sampling/importance_sampling_ratio/min": 0.014635493978857994, |
| "sampling/sampling_logp_difference/max": 4.224305629730225, |
| "sampling/sampling_logp_difference/mean": 0.017568625509738922, |
| "step": 139, |
| "step_time": 214.9732713351259 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14780.0, |
| "completions/max_terminated_length": 14780.0, |
| "completions/mean_length": 10126.9375, |
| "completions/mean_terminated_length": 10126.9375, |
| "completions/min_length": 5722.0, |
| "completions/min_terminated_length": 5722.0, |
| "entropy": 0.30815622955560684, |
| "epoch": 0.16806722689075632, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43311142921447754, |
| "learning_rate": 4.44e-07, |
| "loss": 0.0193, |
| "num_tokens": 49251000.0, |
| "reward": 2.31791353225708, |
| "reward_std": 2.4861373901367188, |
| "rewards/reward_func/mean": 2.31791353225708, |
| "rewards/reward_func/std": 2.4861373901367188, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9891684055328369, |
| "sampling/importance_sampling_ratio/min": 0.002064452273771167, |
| "sampling/sampling_logp_difference/max": 6.18289041519165, |
| "sampling/sampling_logp_difference/mean": 0.02135949209332466, |
| "step": 140, |
| "step_time": 213.63927030388732 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15453.0, |
| "completions/max_terminated_length": 15453.0, |
| "completions/mean_length": 9498.9375, |
| "completions/mean_terminated_length": 9498.9375, |
| "completions/min_length": 5658.0, |
| "completions/min_terminated_length": 5658.0, |
| "entropy": 0.23229695297777653, |
| "epoch": 0.16926770708283315, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.30538368225097656, |
| "learning_rate": 4.3999999999999997e-07, |
| "loss": -0.0339, |
| "num_tokens": 49601214.0, |
| "reward": 0.6548795700073242, |
| "reward_std": 0.8164750337600708, |
| "rewards/reward_func/mean": 0.6548795700073242, |
| "rewards/reward_func/std": 0.8164750337600708, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2676289081573486, |
| "sampling/importance_sampling_ratio/mean": 0.9919700622558594, |
| "sampling/importance_sampling_ratio/min": 0.0972149670124054, |
| "sampling/sampling_logp_difference/max": 2.3308305740356445, |
| "sampling/sampling_logp_difference/mean": 0.016680430620908737, |
| "step": 141, |
| "step_time": 231.1800272183027 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14022.0, |
| "completions/max_terminated_length": 14022.0, |
| "completions/mean_length": 8557.125, |
| "completions/mean_terminated_length": 8557.125, |
| "completions/min_length": 4786.0, |
| "completions/min_terminated_length": 4786.0, |
| "entropy": 0.27586540672928095, |
| "epoch": 0.17046818727490998, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3966992199420929, |
| "learning_rate": 4.36e-07, |
| "loss": -0.0055, |
| "num_tokens": 49905418.0, |
| "reward": 0.8080017566680908, |
| "reward_std": 1.4851527214050293, |
| "rewards/reward_func/mean": 0.8080017566680908, |
| "rewards/reward_func/std": 1.4851524829864502, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.329014539718628, |
| "sampling/importance_sampling_ratio/mean": 0.9903725981712341, |
| "sampling/importance_sampling_ratio/min": 0.3647924065589905, |
| "sampling/sampling_logp_difference/max": 1.0084269046783447, |
| "sampling/sampling_logp_difference/mean": 0.01935010962188244, |
| "step": 142, |
| "step_time": 201.8291045529768 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10552.0, |
| "completions/max_terminated_length": 10552.0, |
| "completions/mean_length": 8153.9375, |
| "completions/mean_terminated_length": 8153.9375, |
| "completions/min_length": 5045.0, |
| "completions/min_terminated_length": 5045.0, |
| "entropy": 0.260047709569335, |
| "epoch": 0.1716686674669868, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3805393874645233, |
| "learning_rate": 4.3199999999999995e-07, |
| "loss": -0.0278, |
| "num_tokens": 50202872.0, |
| "reward": 0.9525775909423828, |
| "reward_std": 1.2389754056930542, |
| "rewards/reward_func/mean": 0.9525775909423828, |
| "rewards/reward_func/std": 1.2389752864837646, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.504868507385254, |
| "sampling/importance_sampling_ratio/mean": 0.9909122586250305, |
| "sampling/importance_sampling_ratio/min": 0.055922865867614746, |
| "sampling/sampling_logp_difference/max": 2.883781909942627, |
| "sampling/sampling_logp_difference/mean": 0.018256302922964096, |
| "step": 143, |
| "step_time": 171.05576233169995 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19289.0, |
| "completions/max_terminated_length": 19289.0, |
| "completions/mean_length": 10663.5625, |
| "completions/mean_terminated_length": 10663.5625, |
| "completions/min_length": 5378.0, |
| "completions/min_terminated_length": 5378.0, |
| "entropy": 0.2466154731810093, |
| "epoch": 0.17286914765906364, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34512680768966675, |
| "learning_rate": 4.2799999999999997e-07, |
| "loss": 0.0035, |
| "num_tokens": 50586930.0, |
| "reward": 2.0170817375183105, |
| "reward_std": 5.622786998748779, |
| "rewards/reward_func/mean": 2.039245128631592, |
| "rewards/reward_func/std": 5.613083362579346, |
| "rewards/soft_overlong_punishment_reward/mean": -0.02216339111328125, |
| "rewards/soft_overlong_punishment_reward/std": 0.12537507712841034, |
| "sampling/importance_sampling_ratio/max": 2.3612794876098633, |
| "sampling/importance_sampling_ratio/mean": 0.9915375113487244, |
| "sampling/importance_sampling_ratio/min": 0.20737797021865845, |
| "sampling/sampling_logp_difference/max": 1.5732121467590332, |
| "sampling/sampling_logp_difference/mean": 0.017183100804686546, |
| "step": 144, |
| "step_time": 274.66985757567454 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14072.0, |
| "completions/max_terminated_length": 14072.0, |
| "completions/mean_length": 10114.15625, |
| "completions/mean_terminated_length": 10114.15625, |
| "completions/min_length": 5891.0, |
| "completions/min_terminated_length": 5891.0, |
| "entropy": 0.2593291876837611, |
| "epoch": 0.17406962785114047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4165084958076477, |
| "learning_rate": 4.24e-07, |
| "loss": -0.0155, |
| "num_tokens": 50952575.0, |
| "reward": 0.6612014174461365, |
| "reward_std": 0.6770195364952087, |
| "rewards/reward_func/mean": 0.6612014174461365, |
| "rewards/reward_func/std": 0.6770195364952087, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3497302532196045, |
| "sampling/importance_sampling_ratio/mean": 0.9909900426864624, |
| "sampling/importance_sampling_ratio/min": 0.00037760011036880314, |
| "sampling/sampling_logp_difference/max": 7.881674766540527, |
| "sampling/sampling_logp_difference/mean": 0.018293218687176704, |
| "step": 145, |
| "step_time": 220.56290533288848 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15097.0, |
| "completions/max_terminated_length": 15097.0, |
| "completions/mean_length": 10543.625, |
| "completions/mean_terminated_length": 10543.625, |
| "completions/min_length": 5699.0, |
| "completions/min_terminated_length": 5699.0, |
| "entropy": 0.2614777237176895, |
| "epoch": 0.1752701080432173, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3072114884853363, |
| "learning_rate": 4.1999999999999995e-07, |
| "loss": 0.0314, |
| "num_tokens": 51330571.0, |
| "reward": 0.5684531927108765, |
| "reward_std": 0.9393928647041321, |
| "rewards/reward_func/mean": 0.5684531927108765, |
| "rewards/reward_func/std": 0.9393928050994873, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.8473550081253052, |
| "sampling/importance_sampling_ratio/mean": 0.9907210469245911, |
| "sampling/importance_sampling_ratio/min": 0.07700582593679428, |
| "sampling/sampling_logp_difference/max": 2.5638742446899414, |
| "sampling/sampling_logp_difference/mean": 0.01846243068575859, |
| "step": 146, |
| "step_time": 453.85375829122495 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13158.0, |
| "completions/max_terminated_length": 13158.0, |
| "completions/mean_length": 8469.71875, |
| "completions/mean_terminated_length": 8469.71875, |
| "completions/min_length": 3331.0, |
| "completions/min_terminated_length": 3331.0, |
| "entropy": 0.29093294590711594, |
| "epoch": 0.17647058823529413, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4070035517215729, |
| "learning_rate": 4.1599999999999997e-07, |
| "loss": 0.0461, |
| "num_tokens": 51637050.0, |
| "reward": 2.5153932571411133, |
| "reward_std": 2.769888401031494, |
| "rewards/reward_func/mean": 2.5153932571411133, |
| "rewards/reward_func/std": 2.769888401031494, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0668880939483643, |
| "sampling/importance_sampling_ratio/mean": 0.9898422956466675, |
| "sampling/importance_sampling_ratio/min": 0.24710887670516968, |
| "sampling/sampling_logp_difference/max": 1.3979263305664062, |
| "sampling/sampling_logp_difference/mean": 0.0202447809278965, |
| "step": 147, |
| "step_time": 189.9260320410831 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14203.0, |
| "completions/max_terminated_length": 14203.0, |
| "completions/mean_length": 8437.125, |
| "completions/mean_terminated_length": 8437.125, |
| "completions/min_length": 4104.0, |
| "completions/min_terminated_length": 4104.0, |
| "entropy": 0.2681101858615875, |
| "epoch": 0.17767106842737096, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.39123448729515076, |
| "learning_rate": 4.12e-07, |
| "loss": 0.022, |
| "num_tokens": 51937598.0, |
| "reward": 2.7620649337768555, |
| "reward_std": 3.608797073364258, |
| "rewards/reward_func/mean": 2.7620649337768555, |
| "rewards/reward_func/std": 3.608797073364258, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.5222227573394775, |
| "sampling/importance_sampling_ratio/mean": 0.9908010959625244, |
| "sampling/importance_sampling_ratio/min": 0.23630328476428986, |
| "sampling/sampling_logp_difference/max": 1.4426392316818237, |
| "sampling/sampling_logp_difference/mean": 0.018574317917227745, |
| "step": 148, |
| "step_time": 205.4168942749966 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17113.0, |
| "completions/max_terminated_length": 17113.0, |
| "completions/mean_length": 9732.375, |
| "completions/mean_terminated_length": 9732.375, |
| "completions/min_length": 5745.0, |
| "completions/min_terminated_length": 5745.0, |
| "entropy": 0.28236152697354555, |
| "epoch": 0.1788715486194478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41433754563331604, |
| "learning_rate": 4.0799999999999995e-07, |
| "loss": -0.0364, |
| "num_tokens": 52280586.0, |
| "reward": 1.7899069786071777, |
| "reward_std": 2.3258073329925537, |
| "rewards/reward_func/mean": 1.799375057220459, |
| "rewards/reward_func/std": 2.324876070022583, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00946807861328125, |
| "rewards/soft_overlong_punishment_reward/std": 0.03785913065075874, |
| "sampling/importance_sampling_ratio/max": 1.7942250967025757, |
| "sampling/importance_sampling_ratio/mean": 0.9903931021690369, |
| "sampling/importance_sampling_ratio/min": 0.028741903603076935, |
| "sampling/sampling_logp_difference/max": 3.5493991374969482, |
| "sampling/sampling_logp_difference/mean": 0.01913137547671795, |
| "step": 149, |
| "step_time": 246.32968526915647 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11989.0, |
| "completions/max_terminated_length": 11989.0, |
| "completions/mean_length": 6893.03125, |
| "completions/mean_terminated_length": 6893.03125, |
| "completions/min_length": 2952.0, |
| "completions/min_terminated_length": 2952.0, |
| "entropy": 0.3159897867590189, |
| "epoch": 0.18007202881152462, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4671184718608856, |
| "learning_rate": 4.04e-07, |
| "loss": 0.022, |
| "num_tokens": 52520571.0, |
| "reward": 2.152081251144409, |
| "reward_std": 2.514601707458496, |
| "rewards/reward_func/mean": 2.152081251144409, |
| "rewards/reward_func/std": 2.514601707458496, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3258213996887207, |
| "sampling/importance_sampling_ratio/mean": 0.9888448715209961, |
| "sampling/importance_sampling_ratio/min": 0.2504069209098816, |
| "sampling/sampling_logp_difference/max": 1.384667992591858, |
| "sampling/sampling_logp_difference/mean": 0.021897951140999794, |
| "step": 150, |
| "step_time": 195.26920710073318 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12887.0, |
| "completions/max_terminated_length": 12887.0, |
| "completions/mean_length": 8146.09375, |
| "completions/mean_terminated_length": 8146.09375, |
| "completions/min_length": 4733.0, |
| "completions/min_terminated_length": 4733.0, |
| "entropy": 0.2663646712899208, |
| "epoch": 0.18127250900360145, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4627015292644501, |
| "learning_rate": 4e-07, |
| "loss": -0.0405, |
| "num_tokens": 52819142.0, |
| "reward": 1.819735050201416, |
| "reward_std": 2.4615325927734375, |
| "rewards/reward_func/mean": 1.819735050201416, |
| "rewards/reward_func/std": 2.4615325927734375, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3554580211639404, |
| "sampling/importance_sampling_ratio/mean": 0.9907447695732117, |
| "sampling/importance_sampling_ratio/min": 0.12350412458181381, |
| "sampling/sampling_logp_difference/max": 2.0914807319641113, |
| "sampling/sampling_logp_difference/mean": 0.01854807324707508, |
| "step": 151, |
| "step_time": 201.55244890286122 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16359.0, |
| "completions/max_terminated_length": 16359.0, |
| "completions/mean_length": 9608.84375, |
| "completions/mean_terminated_length": 9608.84375, |
| "completions/min_length": 4375.0, |
| "completions/min_terminated_length": 4375.0, |
| "entropy": 0.2643621703609824, |
| "epoch": 0.18247298919567828, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4149344563484192, |
| "learning_rate": 3.96e-07, |
| "loss": 0.0057, |
| "num_tokens": 53164609.0, |
| "reward": 0.8813707828521729, |
| "reward_std": 1.7989546060562134, |
| "rewards/reward_func/mean": 0.8813707828521729, |
| "rewards/reward_func/std": 1.7989544868469238, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.488729476928711, |
| "sampling/importance_sampling_ratio/mean": 0.9908096790313721, |
| "sampling/importance_sampling_ratio/min": 0.10281366854906082, |
| "sampling/sampling_logp_difference/max": 2.274837017059326, |
| "sampling/sampling_logp_difference/mean": 0.018455805256962776, |
| "step": 152, |
| "step_time": 226.86713050387334 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 10407.0, |
| "completions/mean_length": 8275.34375, |
| "completions/mean_terminated_length": 7881.64501953125, |
| "completions/min_length": 5786.0, |
| "completions/min_terminated_length": 5786.0, |
| "entropy": 0.2822192329913378, |
| "epoch": 0.1836734693877551, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38609081506729126, |
| "learning_rate": 3.92e-07, |
| "loss": -0.0484, |
| "num_tokens": 53458380.0, |
| "reward": 2.3562521934509277, |
| "reward_std": 3.1254520416259766, |
| "rewards/reward_func/mean": 2.4645183086395264, |
| "rewards/reward_func/std": 3.11552095413208, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, |
| "sampling/importance_sampling_ratio/max": 1.7898873090744019, |
| "sampling/importance_sampling_ratio/mean": 0.990231454372406, |
| "sampling/importance_sampling_ratio/min": 0.25040626525878906, |
| "sampling/sampling_logp_difference/max": 1.384670615196228, |
| "sampling/sampling_logp_difference/mean": 0.019599847495555878, |
| "step": 153, |
| "step_time": 236.71146991685964 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14804.0, |
| "completions/max_terminated_length": 14804.0, |
| "completions/mean_length": 9698.5625, |
| "completions/mean_terminated_length": 9698.5625, |
| "completions/min_length": 4843.0, |
| "completions/min_terminated_length": 4843.0, |
| "entropy": 0.2661178121343255, |
| "epoch": 0.18487394957983194, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38603511452674866, |
| "learning_rate": 3.88e-07, |
| "loss": -0.0049, |
| "num_tokens": 53810886.0, |
| "reward": 0.7766771912574768, |
| "reward_std": 1.2446486949920654, |
| "rewards/reward_func/mean": 0.7766771912574768, |
| "rewards/reward_func/std": 1.2446486949920654, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.604177951812744, |
| "sampling/importance_sampling_ratio/mean": 0.9907273054122925, |
| "sampling/importance_sampling_ratio/min": 2.992864028783515e-05, |
| "sampling/sampling_logp_difference/max": 10.416694641113281, |
| "sampling/sampling_logp_difference/mean": 0.018732445314526558, |
| "step": 154, |
| "step_time": 311.80545009789057 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13282.0, |
| "completions/max_terminated_length": 13282.0, |
| "completions/mean_length": 10236.65625, |
| "completions/mean_terminated_length": 10236.65625, |
| "completions/min_length": 6620.0, |
| "completions/min_terminated_length": 6620.0, |
| "entropy": 0.23385445773601532, |
| "epoch": 0.18607442977190877, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34655675292015076, |
| "learning_rate": 3.84e-07, |
| "loss": 0.0323, |
| "num_tokens": 54183971.0, |
| "reward": 1.4783434867858887, |
| "reward_std": 4.052839756011963, |
| "rewards/reward_func/mean": 1.4783434867858887, |
| "rewards/reward_func/std": 4.052839756011963, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1445651054382324, |
| "sampling/importance_sampling_ratio/mean": 0.9918794631958008, |
| "sampling/importance_sampling_ratio/min": 0.05829664319753647, |
| "sampling/sampling_logp_difference/max": 2.8422107696533203, |
| "sampling/sampling_logp_difference/mean": 0.0164381992071867, |
| "step": 155, |
| "step_time": 215.27548193011899 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13371.0, |
| "completions/max_terminated_length": 13371.0, |
| "completions/mean_length": 8997.625, |
| "completions/mean_terminated_length": 8997.625, |
| "completions/min_length": 5840.0, |
| "completions/min_terminated_length": 5840.0, |
| "entropy": 0.23230466898530722, |
| "epoch": 0.1872749099639856, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3713797628879547, |
| "learning_rate": 3.7999999999999996e-07, |
| "loss": 0.0149, |
| "num_tokens": 54512903.0, |
| "reward": 1.508521318435669, |
| "reward_std": 2.5881083011627197, |
| "rewards/reward_func/mean": 1.508521318435669, |
| "rewards/reward_func/std": 2.5881083011627197, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9918398857116699, |
| "sampling/importance_sampling_ratio/min": 0.12671518325805664, |
| "sampling/sampling_logp_difference/max": 2.0658133029937744, |
| "sampling/sampling_logp_difference/mean": 0.016411125659942627, |
| "step": 156, |
| "step_time": 255.3710762731498 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12916.0, |
| "completions/max_terminated_length": 12916.0, |
| "completions/mean_length": 8303.0625, |
| "completions/mean_terminated_length": 8303.0625, |
| "completions/min_length": 3105.0, |
| "completions/min_terminated_length": 3105.0, |
| "entropy": 0.2881615702062845, |
| "epoch": 0.18847539015606243, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39465340971946716, |
| "learning_rate": 3.76e-07, |
| "loss": 0.0027, |
| "num_tokens": 54806233.0, |
| "reward": 2.848686456680298, |
| "reward_std": 3.52091646194458, |
| "rewards/reward_func/mean": 2.848686456680298, |
| "rewards/reward_func/std": 3.520916700363159, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2623817920684814, |
| "sampling/importance_sampling_ratio/mean": 0.9897094964981079, |
| "sampling/importance_sampling_ratio/min": 0.13919180631637573, |
| "sampling/sampling_logp_difference/max": 1.9719023704528809, |
| "sampling/sampling_logp_difference/mean": 0.020325226709246635, |
| "step": 157, |
| "step_time": 223.82761657808442 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13680.0, |
| "completions/max_terminated_length": 13680.0, |
| "completions/mean_length": 8154.1875, |
| "completions/mean_terminated_length": 8154.1875, |
| "completions/min_length": 4297.0, |
| "completions/min_terminated_length": 4297.0, |
| "entropy": 0.31936432234942913, |
| "epoch": 0.18967587034813926, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.40818971395492554, |
| "learning_rate": 3.72e-07, |
| "loss": -0.0041, |
| "num_tokens": 55087247.0, |
| "reward": 0.5234838724136353, |
| "reward_std": 0.6490806937217712, |
| "rewards/reward_func/mean": 0.5234838724136353, |
| "rewards/reward_func/std": 0.6490806341171265, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9888770580291748, |
| "sampling/importance_sampling_ratio/min": 0.18931546807289124, |
| "sampling/sampling_logp_difference/max": 1.6643404960632324, |
| "sampling/sampling_logp_difference/mean": 0.021932730451226234, |
| "step": 158, |
| "step_time": 218.13758707989473 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15527.0, |
| "completions/max_terminated_length": 15527.0, |
| "completions/mean_length": 10728.09375, |
| "completions/mean_terminated_length": 10728.09375, |
| "completions/min_length": 4513.0, |
| "completions/min_terminated_length": 4513.0, |
| "entropy": 0.3004476949572563, |
| "epoch": 0.1908763505402161, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3766877353191376, |
| "learning_rate": 3.6799999999999996e-07, |
| "loss": 0.0295, |
| "num_tokens": 55460634.0, |
| "reward": 18.52477264404297, |
| "reward_std": 40.702205657958984, |
| "rewards/reward_func/mean": 18.52477264404297, |
| "rewards/reward_func/std": 40.702205657958984, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.10101580619812, |
| "sampling/importance_sampling_ratio/mean": 0.9893911480903625, |
| "sampling/importance_sampling_ratio/min": 0.34575095772743225, |
| "sampling/sampling_logp_difference/max": 1.0620365142822266, |
| "sampling/sampling_logp_difference/mean": 0.021128207445144653, |
| "step": 159, |
| "step_time": 454.3971255911747 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16856.0, |
| "completions/max_terminated_length": 16856.0, |
| "completions/mean_length": 10528.46875, |
| "completions/mean_terminated_length": 10528.46875, |
| "completions/min_length": 7584.0, |
| "completions/min_terminated_length": 7584.0, |
| "entropy": 0.24533494841307402, |
| "epoch": 0.19207683073229292, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3707125782966614, |
| "learning_rate": 3.64e-07, |
| "loss": 0.04, |
| "num_tokens": 55848537.0, |
| "reward": 1.3989126682281494, |
| "reward_std": 0.9870246648788452, |
| "rewards/reward_func/mean": 1.4025137424468994, |
| "rewards/reward_func/std": 0.9815170168876648, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00360107421875, |
| "rewards/soft_overlong_punishment_reward/std": 0.02037075161933899, |
| "sampling/importance_sampling_ratio/max": 2.11053729057312, |
| "sampling/importance_sampling_ratio/mean": 0.9915527105331421, |
| "sampling/importance_sampling_ratio/min": 0.2704220414161682, |
| "sampling/sampling_logp_difference/max": 1.3077714443206787, |
| "sampling/sampling_logp_difference/mean": 0.017412563785910606, |
| "step": 160, |
| "step_time": 272.99579531908967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10456.0, |
| "completions/max_terminated_length": 10456.0, |
| "completions/mean_length": 6813.90625, |
| "completions/mean_terminated_length": 6813.90625, |
| "completions/min_length": 4551.0, |
| "completions/min_terminated_length": 4551.0, |
| "entropy": 0.27734176721423864, |
| "epoch": 0.19327731092436976, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.48373842239379883, |
| "learning_rate": 3.6e-07, |
| "loss": -0.0433, |
| "num_tokens": 56091246.0, |
| "reward": 0.9251768589019775, |
| "reward_std": 1.2734801769256592, |
| "rewards/reward_func/mean": 0.9251768589019775, |
| "rewards/reward_func/std": 1.2734800577163696, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.068244218826294, |
| "sampling/importance_sampling_ratio/mean": 0.990391194820404, |
| "sampling/importance_sampling_ratio/min": 0.18318888545036316, |
| "sampling/sampling_logp_difference/max": 1.697237491607666, |
| "sampling/sampling_logp_difference/mean": 0.01954927295446396, |
| "step": 161, |
| "step_time": 222.0121338494355 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15822.0, |
| "completions/max_terminated_length": 15822.0, |
| "completions/mean_length": 9338.15625, |
| "completions/mean_terminated_length": 9338.15625, |
| "completions/min_length": 4551.0, |
| "completions/min_terminated_length": 4551.0, |
| "entropy": 0.25283763464540243, |
| "epoch": 0.19447779111644659, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3540862202644348, |
| "learning_rate": 3.5599999999999996e-07, |
| "loss": -0.0236, |
| "num_tokens": 56431251.0, |
| "reward": 2.1091599464416504, |
| "reward_std": 3.2207794189453125, |
| "rewards/reward_func/mean": 2.1091599464416504, |
| "rewards/reward_func/std": 3.2207798957824707, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2128798961639404, |
| "sampling/importance_sampling_ratio/mean": 0.9914829730987549, |
| "sampling/importance_sampling_ratio/min": 0.43283000588417053, |
| "sampling/sampling_logp_difference/max": 0.8374102115631104, |
| "sampling/sampling_logp_difference/mean": 0.017802974209189415, |
| "step": 162, |
| "step_time": 242.35264018306043 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13122.0, |
| "completions/max_terminated_length": 13122.0, |
| "completions/mean_length": 9301.21875, |
| "completions/mean_terminated_length": 9301.21875, |
| "completions/min_length": 6263.0, |
| "completions/min_terminated_length": 6263.0, |
| "entropy": 0.2940525095909834, |
| "epoch": 0.19567827130852342, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39872726798057556, |
| "learning_rate": 3.52e-07, |
| "loss": 0.0262, |
| "num_tokens": 56764610.0, |
| "reward": 1.4544447660446167, |
| "reward_std": 1.857795238494873, |
| "rewards/reward_func/mean": 1.4544447660446167, |
| "rewards/reward_func/std": 1.857795238494873, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9896678924560547, |
| "sampling/importance_sampling_ratio/min": 0.36473485827445984, |
| "sampling/sampling_logp_difference/max": 1.1077795028686523, |
| "sampling/sampling_logp_difference/mean": 0.020575741305947304, |
| "step": 163, |
| "step_time": 434.3237003698014 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13825.0, |
| "completions/max_terminated_length": 13825.0, |
| "completions/mean_length": 9424.21875, |
| "completions/mean_terminated_length": 9424.21875, |
| "completions/min_length": 5781.0, |
| "completions/min_terminated_length": 5781.0, |
| "entropy": 0.3086765222251415, |
| "epoch": 0.19687875150060025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40642544627189636, |
| "learning_rate": 3.4799999999999994e-07, |
| "loss": 0.034, |
| "num_tokens": 57091961.0, |
| "reward": 66378.796875, |
| "reward_std": 375489.625, |
| "rewards/reward_func/mean": 66378.796875, |
| "rewards/reward_func/std": 375489.625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.98918217420578, |
| "sampling/importance_sampling_ratio/min": 0.039938684552907944, |
| "sampling/sampling_logp_difference/max": 3.220409870147705, |
| "sampling/sampling_logp_difference/mean": 0.02147614024579525, |
| "step": 164, |
| "step_time": 258.5584741720231 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16400.0, |
| "completions/max_terminated_length": 16400.0, |
| "completions/mean_length": 9196.0, |
| "completions/mean_terminated_length": 9196.0, |
| "completions/min_length": 3739.0, |
| "completions/min_terminated_length": 3739.0, |
| "entropy": 0.2833999376744032, |
| "epoch": 0.19807923169267708, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0096226930618286, |
| "learning_rate": 3.4399999999999996e-07, |
| "loss": -0.0008, |
| "num_tokens": 57423017.0, |
| "reward": 0.45536255836486816, |
| "reward_std": 0.5211973190307617, |
| "rewards/reward_func/mean": 0.45548462867736816, |
| "rewards/reward_func/std": 0.521098792552948, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0001220703125, |
| "rewards/soft_overlong_punishment_reward/std": 0.0006905339541845024, |
| "sampling/importance_sampling_ratio/max": 2.0532639026641846, |
| "sampling/importance_sampling_ratio/mean": 0.9901537895202637, |
| "sampling/importance_sampling_ratio/min": 0.2596132457256317, |
| "sampling/sampling_logp_difference/max": 1.348562240600586, |
| "sampling/sampling_logp_difference/mean": 0.019828852266073227, |
| "step": 165, |
| "step_time": 268.0153879421996 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15331.0, |
| "completions/max_terminated_length": 15331.0, |
| "completions/mean_length": 10289.25, |
| "completions/mean_terminated_length": 10289.25, |
| "completions/min_length": 7699.0, |
| "completions/min_terminated_length": 7699.0, |
| "entropy": 0.2959921294823289, |
| "epoch": 0.1992797118847539, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3890758454799652, |
| "learning_rate": 3.4000000000000003e-07, |
| "loss": -0.0129, |
| "num_tokens": 57786057.0, |
| "reward": 0.2713179886341095, |
| "reward_std": 0.6150152087211609, |
| "rewards/reward_func/mean": 0.2713179886341095, |
| "rewards/reward_func/std": 0.6150152087211609, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9894802570343018, |
| "sampling/importance_sampling_ratio/min": 0.03638508915901184, |
| "sampling/sampling_logp_difference/max": 3.313596248626709, |
| "sampling/sampling_logp_difference/mean": 0.02085329219698906, |
| "step": 166, |
| "step_time": 255.01892217074055 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16600.0, |
| "completions/max_terminated_length": 16600.0, |
| "completions/mean_length": 9496.1875, |
| "completions/mean_terminated_length": 9496.1875, |
| "completions/min_length": 5800.0, |
| "completions/min_terminated_length": 5800.0, |
| "entropy": 0.2720292443409562, |
| "epoch": 0.20048019207683074, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3967978358268738, |
| "learning_rate": 3.36e-07, |
| "loss": 0.0574, |
| "num_tokens": 58128367.0, |
| "reward": 2.866912841796875, |
| "reward_std": 3.3676812648773193, |
| "rewards/reward_func/mean": 2.868560791015625, |
| "rewards/reward_func/std": 3.367079496383667, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00164794921875, |
| "rewards/soft_overlong_punishment_reward/std": 0.009322208352386951, |
| "sampling/importance_sampling_ratio/max": 2.074486494064331, |
| "sampling/importance_sampling_ratio/mean": 0.9905571937561035, |
| "sampling/importance_sampling_ratio/min": 0.25333765149116516, |
| "sampling/sampling_logp_difference/max": 1.3730320930480957, |
| "sampling/sampling_logp_difference/mean": 0.019127877429127693, |
| "step": 167, |
| "step_time": 239.48420156200882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12687.0, |
| "completions/max_terminated_length": 12687.0, |
| "completions/mean_length": 8358.1875, |
| "completions/mean_terminated_length": 8358.1875, |
| "completions/min_length": 5265.0, |
| "completions/min_terminated_length": 5265.0, |
| "entropy": 0.28812102414667606, |
| "epoch": 0.20168067226890757, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41675513982772827, |
| "learning_rate": 3.32e-07, |
| "loss": 0.0333, |
| "num_tokens": 58430437.0, |
| "reward": 2.597444534301758, |
| "reward_std": 3.35589861869812, |
| "rewards/reward_func/mean": 2.597444534301758, |
| "rewards/reward_func/std": 3.355898857116699, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.186401128768921, |
| "sampling/importance_sampling_ratio/mean": 0.9898570775985718, |
| "sampling/importance_sampling_ratio/min": 0.055154554545879364, |
| "sampling/sampling_logp_difference/max": 2.897615909576416, |
| "sampling/sampling_logp_difference/mean": 0.020029492676258087, |
| "step": 168, |
| "step_time": 226.04919587192126 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12104.0, |
| "completions/max_terminated_length": 12104.0, |
| "completions/mean_length": 8018.03125, |
| "completions/mean_terminated_length": 8018.03125, |
| "completions/min_length": 3307.0, |
| "completions/min_terminated_length": 3307.0, |
| "entropy": 0.3021853733807802, |
| "epoch": 0.2028811524609844, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4631645083427429, |
| "learning_rate": 3.28e-07, |
| "loss": 0.0106, |
| "num_tokens": 58713550.0, |
| "reward": 0.6359751224517822, |
| "reward_std": 0.6328838467597961, |
| "rewards/reward_func/mean": 0.6359751224517822, |
| "rewards/reward_func/std": 0.6328837871551514, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1259548664093018, |
| "sampling/importance_sampling_ratio/mean": 0.9894533157348633, |
| "sampling/importance_sampling_ratio/min": 0.005165865179151297, |
| "sampling/sampling_logp_difference/max": 5.265682697296143, |
| "sampling/sampling_logp_difference/mean": 0.020912254229187965, |
| "step": 169, |
| "step_time": 182.4748994258698 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12901.0, |
| "completions/max_terminated_length": 12901.0, |
| "completions/mean_length": 9577.5625, |
| "completions/mean_terminated_length": 9577.5625, |
| "completions/min_length": 5991.0, |
| "completions/min_terminated_length": 5991.0, |
| "entropy": 0.2918419409543276, |
| "epoch": 0.20408163265306123, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4238099157810211, |
| "learning_rate": 3.24e-07, |
| "loss": 0.0158, |
| "num_tokens": 59055152.0, |
| "reward": 1.1994872093200684, |
| "reward_std": 1.4579551219940186, |
| "rewards/reward_func/mean": 1.1994872093200684, |
| "rewards/reward_func/std": 1.457955241203308, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3555917739868164, |
| "sampling/importance_sampling_ratio/mean": 0.9899040460586548, |
| "sampling/importance_sampling_ratio/min": 0.15884944796562195, |
| "sampling/sampling_logp_difference/max": 1.8397983312606812, |
| "sampling/sampling_logp_difference/mean": 0.020075272768735886, |
| "step": 170, |
| "step_time": 205.09224270377308 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11374.0, |
| "completions/max_terminated_length": 11374.0, |
| "completions/mean_length": 7372.3125, |
| "completions/mean_terminated_length": 7372.3125, |
| "completions/min_length": 2447.0, |
| "completions/min_terminated_length": 2447.0, |
| "entropy": 0.2957879761233926, |
| "epoch": 0.20528211284513806, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4720025956630707, |
| "learning_rate": 3.2e-07, |
| "loss": 0.0587, |
| "num_tokens": 59316858.0, |
| "reward": 2.2769434452056885, |
| "reward_std": 2.0495495796203613, |
| "rewards/reward_func/mean": 2.2769434452056885, |
| "rewards/reward_func/std": 2.0495495796203613, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989801287651062, |
| "sampling/importance_sampling_ratio/min": 0.36141398549079895, |
| "sampling/sampling_logp_difference/max": 1.1415172815322876, |
| "sampling/sampling_logp_difference/mean": 0.020276790484786034, |
| "step": 171, |
| "step_time": 171.338580377982 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13035.0, |
| "completions/max_terminated_length": 13035.0, |
| "completions/mean_length": 9139.9375, |
| "completions/mean_terminated_length": 9139.9375, |
| "completions/min_length": 6085.0, |
| "completions/min_terminated_length": 6085.0, |
| "entropy": 0.265663193538785, |
| "epoch": 0.2064825930372149, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4693741202354431, |
| "learning_rate": 3.1599999999999997e-07, |
| "loss": 0.0361, |
| "num_tokens": 59651160.0, |
| "reward": 1.905058741569519, |
| "reward_std": 3.157362699508667, |
| "rewards/reward_func/mean": 1.905058741569519, |
| "rewards/reward_func/std": 3.157362699508667, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.7062559127807617, |
| "sampling/importance_sampling_ratio/mean": 0.9907447695732117, |
| "sampling/importance_sampling_ratio/min": 0.05720826983451843, |
| "sampling/sampling_logp_difference/max": 2.8610568046569824, |
| "sampling/sampling_logp_difference/mean": 0.018653716892004013, |
| "step": 172, |
| "step_time": 397.120246386854 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14177.0, |
| "completions/max_terminated_length": 14177.0, |
| "completions/mean_length": 9426.0625, |
| "completions/mean_terminated_length": 9426.0625, |
| "completions/min_length": 5614.0, |
| "completions/min_terminated_length": 5614.0, |
| "entropy": 0.2500729188323021, |
| "epoch": 0.20768307322929172, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.2702878415584564, |
| "learning_rate": 3.12e-07, |
| "loss": 0.0171, |
| "num_tokens": 59992778.0, |
| "reward": 1.3135590553283691, |
| "reward_std": 2.5076005458831787, |
| "rewards/reward_func/mean": 1.3135590553283691, |
| "rewards/reward_func/std": 2.5076003074645996, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.058269500732422, |
| "sampling/importance_sampling_ratio/mean": 0.9914534687995911, |
| "sampling/importance_sampling_ratio/min": 0.10936449468135834, |
| "sampling/sampling_logp_difference/max": 2.213068962097168, |
| "sampling/sampling_logp_difference/mean": 0.01742243394255638, |
| "step": 173, |
| "step_time": 239.5976502839476 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12497.0, |
| "completions/max_terminated_length": 12497.0, |
| "completions/mean_length": 8828.0, |
| "completions/mean_terminated_length": 8828.0, |
| "completions/min_length": 6421.0, |
| "completions/min_terminated_length": 6421.0, |
| "entropy": 0.27684462163597345, |
| "epoch": 0.20888355342136855, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3942298889160156, |
| "learning_rate": 3.08e-07, |
| "loss": 0.0059, |
| "num_tokens": 60315146.0, |
| "reward": 0.6179296970367432, |
| "reward_std": 0.6792400479316711, |
| "rewards/reward_func/mean": 0.6179296970367432, |
| "rewards/reward_func/std": 0.6792401075363159, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9904294013977051, |
| "sampling/importance_sampling_ratio/min": 0.2878604829311371, |
| "sampling/sampling_logp_difference/max": 1.245279312133789, |
| "sampling/sampling_logp_difference/mean": 0.01934416964650154, |
| "step": 174, |
| "step_time": 214.77978121873457 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17360.0, |
| "completions/max_terminated_length": 17360.0, |
| "completions/mean_length": 9081.84375, |
| "completions/mean_terminated_length": 9081.84375, |
| "completions/min_length": 4942.0, |
| "completions/min_terminated_length": 4942.0, |
| "entropy": 0.2940607778728008, |
| "epoch": 0.21008403361344538, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4100255072116852, |
| "learning_rate": 3.0399999999999997e-07, |
| "loss": 0.0503, |
| "num_tokens": 60635101.0, |
| "reward": 1.2948029041290283, |
| "reward_std": 1.4059300422668457, |
| "rewards/reward_func/mean": 1.3022491931915283, |
| "rewards/reward_func/std": 1.3984326124191284, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0074462890625, |
| "rewards/soft_overlong_punishment_reward/std": 0.04212257266044617, |
| "sampling/importance_sampling_ratio/max": 2.523259162902832, |
| "sampling/importance_sampling_ratio/mean": 0.9897422790527344, |
| "sampling/importance_sampling_ratio/min": 0.2878084182739258, |
| "sampling/sampling_logp_difference/max": 1.2454602718353271, |
| "sampling/sampling_logp_difference/mean": 0.020703017711639404, |
| "step": 175, |
| "step_time": 467.0385398848448 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 20183.0, |
| "completions/mean_length": 11330.46875, |
| "completions/mean_terminated_length": 9219.0390625, |
| "completions/min_length": 6454.0, |
| "completions/min_terminated_length": 6454.0, |
| "entropy": 0.2567200744524598, |
| "epoch": 0.2112845138055222, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.33059075474739075, |
| "learning_rate": 3e-07, |
| "loss": -0.0585, |
| "num_tokens": 61020372.0, |
| "reward": 1.0467814207077026, |
| "reward_std": 1.7194699048995972, |
| "rewards/reward_func/mean": 1.55478835105896, |
| "rewards/reward_func/std": 1.5100483894348145, |
| "rewards/soft_overlong_punishment_reward/mean": -0.21648406982421875, |
| "rewards/soft_overlong_punishment_reward/std": 0.41583752632141113, |
| "sampling/importance_sampling_ratio/max": 2.49721097946167, |
| "sampling/importance_sampling_ratio/mean": 0.9905225038528442, |
| "sampling/importance_sampling_ratio/min": 0.2589978873729706, |
| "sampling/sampling_logp_difference/max": 1.3509353399276733, |
| "sampling/sampling_logp_difference/mean": 0.019211364910006523, |
| "step": 176, |
| "step_time": 262.3387938350206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13843.0, |
| "completions/max_terminated_length": 13843.0, |
| "completions/mean_length": 9019.90625, |
| "completions/mean_terminated_length": 9019.90625, |
| "completions/min_length": 5026.0, |
| "completions/min_terminated_length": 5026.0, |
| "entropy": 0.26270918548107147, |
| "epoch": 0.21248499399759904, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40801480412483215, |
| "learning_rate": 2.9599999999999995e-07, |
| "loss": -0.0516, |
| "num_tokens": 61349169.0, |
| "reward": 0.7610185146331787, |
| "reward_std": 0.6794523000717163, |
| "rewards/reward_func/mean": 0.7610185146331787, |
| "rewards/reward_func/std": 0.6794522404670715, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9909648895263672, |
| "sampling/importance_sampling_ratio/min": 0.45922839641571045, |
| "sampling/sampling_logp_difference/max": 1.4038057327270508, |
| "sampling/sampling_logp_difference/mean": 0.0184706449508667, |
| "step": 177, |
| "step_time": 210.60857710335404 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13779.0, |
| "completions/max_terminated_length": 13779.0, |
| "completions/mean_length": 8056.8125, |
| "completions/mean_terminated_length": 8056.8125, |
| "completions/min_length": 4695.0, |
| "completions/min_terminated_length": 4695.0, |
| "entropy": 0.2336973724886775, |
| "epoch": 0.21368547418967587, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4138389527797699, |
| "learning_rate": 2.9199999999999997e-07, |
| "loss": 0.0636, |
| "num_tokens": 61643739.0, |
| "reward": 0.7230340242385864, |
| "reward_std": 1.2502714395523071, |
| "rewards/reward_func/mean": 0.7230340242385864, |
| "rewards/reward_func/std": 1.2502713203430176, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.6206603050231934, |
| "sampling/importance_sampling_ratio/mean": 0.9920225143432617, |
| "sampling/importance_sampling_ratio/min": 0.001480705919675529, |
| "sampling/sampling_logp_difference/max": 6.5152363777160645, |
| "sampling/sampling_logp_difference/mean": 0.016540315002202988, |
| "step": 178, |
| "step_time": 223.01532334729563 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 20367.0, |
| "completions/max_terminated_length": 20367.0, |
| "completions/mean_length": 9921.0625, |
| "completions/mean_terminated_length": 9921.0625, |
| "completions/min_length": 6193.0, |
| "completions/min_terminated_length": 6193.0, |
| "entropy": 0.2881326600909233, |
| "epoch": 0.2148859543817527, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.34104132652282715, |
| "learning_rate": 2.88e-07, |
| "loss": 0.05, |
| "num_tokens": 61997757.0, |
| "reward": 1.612834095954895, |
| "reward_std": 2.867422580718994, |
| "rewards/reward_func/mean": 1.6432219743728638, |
| "rewards/reward_func/std": 2.8589067459106445, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03038787841796875, |
| "rewards/soft_overlong_punishment_reward/std": 0.17189979553222656, |
| "sampling/importance_sampling_ratio/max": 2.565842866897583, |
| "sampling/importance_sampling_ratio/mean": 0.9899250268936157, |
| "sampling/importance_sampling_ratio/min": 0.11633654683828354, |
| "sampling/sampling_logp_difference/max": 2.1512680053710938, |
| "sampling/sampling_logp_difference/mean": 0.020425235852599144, |
| "step": 179, |
| "step_time": 249.0168090411462 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12388.0, |
| "completions/max_terminated_length": 12388.0, |
| "completions/mean_length": 6812.59375, |
| "completions/mean_terminated_length": 6812.59375, |
| "completions/min_length": 2401.0, |
| "completions/min_terminated_length": 2401.0, |
| "entropy": 0.3213646560907364, |
| "epoch": 0.21608643457382953, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.510503888130188, |
| "learning_rate": 2.8399999999999995e-07, |
| "loss": -0.0106, |
| "num_tokens": 62243096.0, |
| "reward": 1.7355564832687378, |
| "reward_std": 1.7355234622955322, |
| "rewards/reward_func/mean": 1.7355564832687378, |
| "rewards/reward_func/std": 1.7355234622955322, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2783377170562744, |
| "sampling/importance_sampling_ratio/mean": 0.9888811707496643, |
| "sampling/importance_sampling_ratio/min": 0.27378326654434204, |
| "sampling/sampling_logp_difference/max": 1.2954185009002686, |
| "sampling/sampling_logp_difference/mean": 0.022066859528422356, |
| "step": 180, |
| "step_time": 233.19768030114938 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 17498.0, |
| "completions/mean_length": 10577.9375, |
| "completions/mean_terminated_length": 10258.515625, |
| "completions/min_length": 5877.0, |
| "completions/min_terminated_length": 5877.0, |
| "entropy": 0.2779244324192405, |
| "epoch": 0.21728691476590636, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.350612074136734, |
| "learning_rate": 2.8e-07, |
| "loss": -0.0024, |
| "num_tokens": 62611454.0, |
| "reward": 2.4395365715026855, |
| "reward_std": 4.4411797523498535, |
| "rewards/reward_func/mean": 2.479285717010498, |
| "rewards/reward_func/std": 4.416739463806152, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0397491455078125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1816953718662262, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9902195930480957, |
| "sampling/importance_sampling_ratio/min": 0.2922537326812744, |
| "sampling/sampling_logp_difference/max": 1.230132818222046, |
| "sampling/sampling_logp_difference/mean": 0.01920732483267784, |
| "step": 181, |
| "step_time": 265.4475303606596 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12516.0, |
| "completions/max_terminated_length": 12516.0, |
| "completions/mean_length": 8191.75, |
| "completions/mean_terminated_length": 8191.75, |
| "completions/min_length": 4992.0, |
| "completions/min_terminated_length": 4992.0, |
| "entropy": 0.26552946865558624, |
| "epoch": 0.2184873949579832, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4106624126434326, |
| "learning_rate": 2.7600000000000004e-07, |
| "loss": 0.0802, |
| "num_tokens": 62906998.0, |
| "reward": 2.137417793273926, |
| "reward_std": 2.385087013244629, |
| "rewards/reward_func/mean": 2.137417793273926, |
| "rewards/reward_func/std": 2.385087013244629, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4886276721954346, |
| "sampling/importance_sampling_ratio/mean": 0.9908239245414734, |
| "sampling/importance_sampling_ratio/min": 0.23087278008460999, |
| "sampling/sampling_logp_difference/max": 1.465888500213623, |
| "sampling/sampling_logp_difference/mean": 0.018649548292160034, |
| "step": 182, |
| "step_time": 180.85359141172376 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11870.0, |
| "completions/max_terminated_length": 11870.0, |
| "completions/mean_length": 8648.4375, |
| "completions/mean_terminated_length": 8648.4375, |
| "completions/min_length": 5557.0, |
| "completions/min_terminated_length": 5557.0, |
| "entropy": 0.2885238602757454, |
| "epoch": 0.21968787515006002, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4420488178730011, |
| "learning_rate": 2.72e-07, |
| "loss": -0.0295, |
| "num_tokens": 63213556.0, |
| "reward": 0.8226311802864075, |
| "reward_std": 0.6850433945655823, |
| "rewards/reward_func/mean": 0.8226311802864075, |
| "rewards/reward_func/std": 0.6850433945655823, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.508232593536377, |
| "sampling/importance_sampling_ratio/mean": 0.989946722984314, |
| "sampling/importance_sampling_ratio/min": 0.0002403110993327573, |
| "sampling/sampling_logp_difference/max": 8.333576202392578, |
| "sampling/sampling_logp_difference/mean": 0.02029946818947792, |
| "step": 183, |
| "step_time": 198.74373724497855 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15138.0, |
| "completions/max_terminated_length": 15138.0, |
| "completions/mean_length": 10048.90625, |
| "completions/mean_terminated_length": 10048.90625, |
| "completions/min_length": 5452.0, |
| "completions/min_terminated_length": 5452.0, |
| "entropy": 0.24506080709397793, |
| "epoch": 0.22088835534213686, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3524847626686096, |
| "learning_rate": 2.68e-07, |
| "loss": 0.0155, |
| "num_tokens": 63572233.0, |
| "reward": 3.185716152191162, |
| "reward_std": 4.615874290466309, |
| "rewards/reward_func/mean": 3.185716152191162, |
| "rewards/reward_func/std": 4.61587381362915, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9916706085205078, |
| "sampling/importance_sampling_ratio/min": 0.018803074955940247, |
| "sampling/sampling_logp_difference/max": 3.9737348556518555, |
| "sampling/sampling_logp_difference/mean": 0.01703486405313015, |
| "step": 184, |
| "step_time": 224.22839038609527 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11046.0, |
| "completions/max_terminated_length": 11046.0, |
| "completions/mean_length": 7622.9375, |
| "completions/mean_terminated_length": 7622.9375, |
| "completions/min_length": 4460.0, |
| "completions/min_terminated_length": 4460.0, |
| "entropy": 0.2557719862088561, |
| "epoch": 0.22208883553421369, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42476049065589905, |
| "learning_rate": 2.64e-07, |
| "loss": -0.0013, |
| "num_tokens": 63848983.0, |
| "reward": 4.9202985763549805, |
| "reward_std": 5.301763534545898, |
| "rewards/reward_func/mean": 4.9202985763549805, |
| "rewards/reward_func/std": 5.301763534545898, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9607212543487549, |
| "sampling/importance_sampling_ratio/mean": 0.9911612868309021, |
| "sampling/importance_sampling_ratio/min": 0.019983841106295586, |
| "sampling/sampling_logp_difference/max": 3.9128313064575195, |
| "sampling/sampling_logp_difference/mean": 0.017659198492765427, |
| "step": 185, |
| "step_time": 181.76806327118538 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13441.0, |
| "completions/max_terminated_length": 13441.0, |
| "completions/mean_length": 9717.0, |
| "completions/mean_terminated_length": 9717.0, |
| "completions/min_length": 5815.0, |
| "completions/min_terminated_length": 5815.0, |
| "entropy": 0.24156292621046305, |
| "epoch": 0.22328931572629052, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.442635715007782, |
| "learning_rate": 2.6e-07, |
| "loss": -0.0102, |
| "num_tokens": 64210287.0, |
| "reward": 1.863988995552063, |
| "reward_std": 2.8825900554656982, |
| "rewards/reward_func/mean": 1.863988995552063, |
| "rewards/reward_func/std": 2.8825902938842773, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.648536443710327, |
| "sampling/importance_sampling_ratio/mean": 0.9914799928665161, |
| "sampling/importance_sampling_ratio/min": 0.027922935783863068, |
| "sampling/sampling_logp_difference/max": 3.5783069133758545, |
| "sampling/sampling_logp_difference/mean": 0.017204025760293007, |
| "step": 186, |
| "step_time": 238.74721054232214 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13385.0, |
| "completions/max_terminated_length": 13385.0, |
| "completions/mean_length": 10068.6875, |
| "completions/mean_terminated_length": 10068.6875, |
| "completions/min_length": 7343.0, |
| "completions/min_terminated_length": 7343.0, |
| "entropy": 0.2825509188696742, |
| "epoch": 0.22448979591836735, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4277617335319519, |
| "learning_rate": 2.56e-07, |
| "loss": 0.0193, |
| "num_tokens": 64569965.0, |
| "reward": 0.8854573965072632, |
| "reward_std": 0.6112152934074402, |
| "rewards/reward_func/mean": 0.8854573965072632, |
| "rewards/reward_func/std": 0.6112152934074402, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0112133026123047, |
| "sampling/importance_sampling_ratio/mean": 0.9901423454284668, |
| "sampling/importance_sampling_ratio/min": 0.16564348340034485, |
| "sampling/sampling_logp_difference/max": 1.7979174852371216, |
| "sampling/sampling_logp_difference/mean": 0.019815031439065933, |
| "step": 187, |
| "step_time": 228.7444448468741 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13373.0, |
| "completions/max_terminated_length": 13373.0, |
| "completions/mean_length": 9405.59375, |
| "completions/mean_terminated_length": 9405.59375, |
| "completions/min_length": 5967.0, |
| "completions/min_terminated_length": 5967.0, |
| "entropy": 0.318213177844882, |
| "epoch": 0.22569027611044418, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.33108896017074585, |
| "learning_rate": 2.52e-07, |
| "loss": 0.0167, |
| "num_tokens": 64897776.0, |
| "reward": 0.5405811071395874, |
| "reward_std": 0.6606644988059998, |
| "rewards/reward_func/mean": 0.5405811071395874, |
| "rewards/reward_func/std": 0.6606644988059998, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9889894723892212, |
| "sampling/importance_sampling_ratio/min": 0.19092479348182678, |
| "sampling/sampling_logp_difference/max": 1.9465055465698242, |
| "sampling/sampling_logp_difference/mean": 0.02206978015601635, |
| "step": 188, |
| "step_time": 415.85145999782253 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11114.0, |
| "completions/max_terminated_length": 11114.0, |
| "completions/mean_length": 8580.78125, |
| "completions/mean_terminated_length": 8580.78125, |
| "completions/min_length": 4516.0, |
| "completions/min_terminated_length": 4516.0, |
| "entropy": 0.3377720918506384, |
| "epoch": 0.226890756302521, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42674168944358826, |
| "learning_rate": 2.48e-07, |
| "loss": 0.0088, |
| "num_tokens": 65198441.0, |
| "reward": 1.591444730758667, |
| "reward_std": 2.4525747299194336, |
| "rewards/reward_func/mean": 1.591444730758667, |
| "rewards/reward_func/std": 2.4525744915008545, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.6534841060638428, |
| "sampling/importance_sampling_ratio/mean": 0.9881213903427124, |
| "sampling/importance_sampling_ratio/min": 0.33775991201400757, |
| "sampling/sampling_logp_difference/max": 1.0854198932647705, |
| "sampling/sampling_logp_difference/mean": 0.022877736017107964, |
| "step": 189, |
| "step_time": 285.5526141769951 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10786.0, |
| "completions/max_terminated_length": 10786.0, |
| "completions/mean_length": 7476.28125, |
| "completions/mean_terminated_length": 7476.28125, |
| "completions/min_length": 4472.0, |
| "completions/min_terminated_length": 4472.0, |
| "entropy": 0.3017116552218795, |
| "epoch": 0.22809123649459784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41298624873161316, |
| "learning_rate": 2.4399999999999996e-07, |
| "loss": 0.049, |
| "num_tokens": 65463538.0, |
| "reward": 1.0010404586791992, |
| "reward_std": 0.7883979678153992, |
| "rewards/reward_func/mean": 1.0010404586791992, |
| "rewards/reward_func/std": 0.7883979082107544, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1348612308502197, |
| "sampling/importance_sampling_ratio/mean": 0.989497184753418, |
| "sampling/importance_sampling_ratio/min": 0.0010880796471610665, |
| "sampling/sampling_logp_difference/max": 6.823340892791748, |
| "sampling/sampling_logp_difference/mean": 0.020549368113279343, |
| "step": 190, |
| "step_time": 186.38824158336502 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11019.0, |
| "completions/max_terminated_length": 11019.0, |
| "completions/mean_length": 7058.15625, |
| "completions/mean_terminated_length": 7058.15625, |
| "completions/min_length": 2793.0, |
| "completions/min_terminated_length": 2793.0, |
| "entropy": 0.29681937769055367, |
| "epoch": 0.22929171668667467, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.48941799998283386, |
| "learning_rate": 2.4e-07, |
| "loss": -0.0289, |
| "num_tokens": 65715367.0, |
| "reward": 2.424999475479126, |
| "reward_std": 2.926403045654297, |
| "rewards/reward_func/mean": 2.424999475479126, |
| "rewards/reward_func/std": 2.926403045654297, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.788496494293213, |
| "sampling/importance_sampling_ratio/mean": 0.9895862936973572, |
| "sampling/importance_sampling_ratio/min": 0.3045606315135956, |
| "sampling/sampling_logp_difference/max": 1.1888850927352905, |
| "sampling/sampling_logp_difference/mean": 0.02054070681333542, |
| "step": 191, |
| "step_time": 202.2660656850785 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15425.0, |
| "completions/max_terminated_length": 15425.0, |
| "completions/mean_length": 10237.875, |
| "completions/mean_terminated_length": 10237.875, |
| "completions/min_length": 4844.0, |
| "completions/min_terminated_length": 4844.0, |
| "entropy": 0.28149267937988043, |
| "epoch": 0.2304921968787515, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5577535033226013, |
| "learning_rate": 2.3599999999999997e-07, |
| "loss": 0.0313, |
| "num_tokens": 66083091.0, |
| "reward": 1.852272391319275, |
| "reward_std": 3.534353494644165, |
| "rewards/reward_func/mean": 1.852272391319275, |
| "rewards/reward_func/std": 3.534353494644165, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9435347318649292, |
| "sampling/importance_sampling_ratio/mean": 0.9903722405433655, |
| "sampling/importance_sampling_ratio/min": 0.42083677649497986, |
| "sampling/sampling_logp_difference/max": 0.8655102252960205, |
| "sampling/sampling_logp_difference/mean": 0.019453980028629303, |
| "step": 192, |
| "step_time": 476.26090911997017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12826.0, |
| "completions/max_terminated_length": 12826.0, |
| "completions/mean_length": 9010.1875, |
| "completions/mean_terminated_length": 9010.1875, |
| "completions/min_length": 5762.0, |
| "completions/min_terminated_length": 5762.0, |
| "entropy": 0.2871004734188318, |
| "epoch": 0.23169267707082833, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4089401364326477, |
| "learning_rate": 2.32e-07, |
| "loss": -0.0462, |
| "num_tokens": 66413569.0, |
| "reward": 34.56924057006836, |
| "reward_std": 156.94326782226562, |
| "rewards/reward_func/mean": 34.56924057006836, |
| "rewards/reward_func/std": 156.9432830810547, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.8682339191436768, |
| "sampling/importance_sampling_ratio/mean": 0.989984929561615, |
| "sampling/importance_sampling_ratio/min": 0.0005362548981793225, |
| "sampling/sampling_logp_difference/max": 7.530900955200195, |
| "sampling/sampling_logp_difference/mean": 0.020170990377664566, |
| "step": 193, |
| "step_time": 259.0703094872879 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10244.0, |
| "completions/max_terminated_length": 10244.0, |
| "completions/mean_length": 7021.78125, |
| "completions/mean_terminated_length": 7021.78125, |
| "completions/min_length": 4505.0, |
| "completions/min_terminated_length": 4505.0, |
| "entropy": 0.281132142059505, |
| "epoch": 0.23289315726290516, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.45947787165641785, |
| "learning_rate": 2.28e-07, |
| "loss": 0.0177, |
| "num_tokens": 66662410.0, |
| "reward": 2.6954410076141357, |
| "reward_std": 2.3101069927215576, |
| "rewards/reward_func/mean": 2.6954410076141357, |
| "rewards/reward_func/std": 2.3101069927215576, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2453787326812744, |
| "sampling/importance_sampling_ratio/mean": 0.9901114702224731, |
| "sampling/importance_sampling_ratio/min": 0.13286785781383514, |
| "sampling/sampling_logp_difference/max": 2.018400192260742, |
| "sampling/sampling_logp_difference/mean": 0.019676901400089264, |
| "step": 194, |
| "step_time": 158.03437204903457 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 14415.0, |
| "completions/mean_length": 10107.21875, |
| "completions/mean_terminated_length": 9772.6123046875, |
| "completions/min_length": 5721.0, |
| "completions/min_terminated_length": 5721.0, |
| "entropy": 0.26805521361529827, |
| "epoch": 0.234093637454982, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3630945682525635, |
| "learning_rate": 2.24e-07, |
| "loss": -0.0655, |
| "num_tokens": 67019769.0, |
| "reward": 0.5480448007583618, |
| "reward_std": 0.7183238863945007, |
| "rewards/reward_func/mean": 0.5979817509651184, |
| "rewards/reward_func/std": 0.6713647246360779, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, |
| "sampling/importance_sampling_ratio/max": 2.4275918006896973, |
| "sampling/importance_sampling_ratio/mean": 0.9905480146408081, |
| "sampling/importance_sampling_ratio/min": 0.3468351662158966, |
| "sampling/sampling_logp_difference/max": 1.0589056015014648, |
| "sampling/sampling_logp_difference/mean": 0.018695417791604996, |
| "step": 195, |
| "step_time": 256.8099763797363 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11044.0, |
| "completions/max_terminated_length": 11044.0, |
| "completions/mean_length": 8487.1875, |
| "completions/mean_terminated_length": 8487.1875, |
| "completions/min_length": 4974.0, |
| "completions/min_terminated_length": 4974.0, |
| "entropy": 0.2542698457837105, |
| "epoch": 0.23529411764705882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40941476821899414, |
| "learning_rate": 2.1999999999999998e-07, |
| "loss": 0.0108, |
| "num_tokens": 67330087.0, |
| "reward": 0.9295157790184021, |
| "reward_std": 1.449187159538269, |
| "rewards/reward_func/mean": 0.9295157790184021, |
| "rewards/reward_func/std": 1.449187159538269, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.435007333755493, |
| "sampling/importance_sampling_ratio/mean": 0.9911128282546997, |
| "sampling/importance_sampling_ratio/min": 0.0041382331401109695, |
| "sampling/sampling_logp_difference/max": 5.487486362457275, |
| "sampling/sampling_logp_difference/mean": 0.01788972318172455, |
| "step": 196, |
| "step_time": 230.38768620893825 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11561.0, |
| "completions/max_terminated_length": 11561.0, |
| "completions/mean_length": 8563.40625, |
| "completions/mean_terminated_length": 8563.40625, |
| "completions/min_length": 3335.0, |
| "completions/min_terminated_length": 3335.0, |
| "entropy": 0.29917307291179895, |
| "epoch": 0.23649459783913565, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.4290471374988556, |
| "learning_rate": 2.1599999999999998e-07, |
| "loss": -0.0069, |
| "num_tokens": 67638532.0, |
| "reward": 0.5732239484786987, |
| "reward_std": 0.6618029475212097, |
| "rewards/reward_func/mean": 0.5732239484786987, |
| "rewards/reward_func/std": 0.6618028879165649, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.300884246826172, |
| "sampling/importance_sampling_ratio/mean": 0.9896308779716492, |
| "sampling/importance_sampling_ratio/min": 0.13782455027103424, |
| "sampling/sampling_logp_difference/max": 1.9817737340927124, |
| "sampling/sampling_logp_difference/mean": 0.020700525492429733, |
| "step": 197, |
| "step_time": 230.13961980782915 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14545.0, |
| "completions/max_terminated_length": 14545.0, |
| "completions/mean_length": 8384.875, |
| "completions/mean_terminated_length": 8384.875, |
| "completions/min_length": 4210.0, |
| "completions/min_terminated_length": 4210.0, |
| "entropy": 0.27590967528522015, |
| "epoch": 0.23769507803121248, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4398875832557678, |
| "learning_rate": 2.12e-07, |
| "loss": 0.019, |
| "num_tokens": 67938368.0, |
| "reward": 29078.322265625, |
| "reward_std": 164479.546875, |
| "rewards/reward_func/mean": 29078.322265625, |
| "rewards/reward_func/std": 164479.546875, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.5150890350341797, |
| "sampling/importance_sampling_ratio/mean": 0.9903858304023743, |
| "sampling/importance_sampling_ratio/min": 0.07026353478431702, |
| "sampling/sampling_logp_difference/max": 2.6555023193359375, |
| "sampling/sampling_logp_difference/mean": 0.019529715180397034, |
| "step": 198, |
| "step_time": 445.3559492408531 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15397.0, |
| "completions/max_terminated_length": 15397.0, |
| "completions/mean_length": 10389.4375, |
| "completions/mean_terminated_length": 10389.4375, |
| "completions/min_length": 6805.0, |
| "completions/min_terminated_length": 6805.0, |
| "entropy": 0.2911730855703354, |
| "epoch": 0.2388955582232893, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42178842425346375, |
| "learning_rate": 2.0799999999999998e-07, |
| "loss": 0.0221, |
| "num_tokens": 68309710.0, |
| "reward": 0.8557122945785522, |
| "reward_std": 0.7761005163192749, |
| "rewards/reward_func/mean": 0.8557122945785522, |
| "rewards/reward_func/std": 0.7761005759239197, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1360371112823486, |
| "sampling/importance_sampling_ratio/mean": 0.989779531955719, |
| "sampling/importance_sampling_ratio/min": 0.04221712425351143, |
| "sampling/sampling_logp_difference/max": 3.1649293899536133, |
| "sampling/sampling_logp_difference/mean": 0.02039753459393978, |
| "step": 199, |
| "step_time": 232.6083577099489 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13338.0, |
| "completions/max_terminated_length": 13338.0, |
| "completions/mean_length": 9125.3125, |
| "completions/mean_terminated_length": 9125.3125, |
| "completions/min_length": 5379.0, |
| "completions/min_terminated_length": 5379.0, |
| "entropy": 0.3312236089259386, |
| "epoch": 0.24009603841536614, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40419086813926697, |
| "learning_rate": 2.0399999999999997e-07, |
| "loss": -0.0259, |
| "num_tokens": 68628320.0, |
| "reward": 0.9386237859725952, |
| "reward_std": 0.6489390730857849, |
| "rewards/reward_func/mean": 0.9386237859725952, |
| "rewards/reward_func/std": 0.6489390134811401, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.085390090942383, |
| "sampling/importance_sampling_ratio/mean": 0.9884665608406067, |
| "sampling/importance_sampling_ratio/min": 0.2759179472923279, |
| "sampling/sampling_logp_difference/max": 1.287651777267456, |
| "sampling/sampling_logp_difference/mean": 0.022430500015616417, |
| "step": 200, |
| "step_time": 201.43533559609205 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11893.0, |
| "completions/max_terminated_length": 11893.0, |
| "completions/mean_length": 7656.96875, |
| "completions/mean_terminated_length": 7656.96875, |
| "completions/min_length": 3211.0, |
| "completions/min_terminated_length": 3211.0, |
| "entropy": 0.2885604901239276, |
| "epoch": 0.24129651860744297, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4403934180736542, |
| "learning_rate": 2e-07, |
| "loss": -0.0085, |
| "num_tokens": 68899535.0, |
| "reward": 2.592362880706787, |
| "reward_std": 2.72981858253479, |
| "rewards/reward_func/mean": 2.592362880706787, |
| "rewards/reward_func/std": 2.72981858253479, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901413917541504, |
| "sampling/importance_sampling_ratio/min": 0.06037893146276474, |
| "sampling/sampling_logp_difference/max": 2.807115077972412, |
| "sampling/sampling_logp_difference/mean": 0.02000146359205246, |
| "step": 201, |
| "step_time": 182.84680638380814 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13383.0, |
| "completions/max_terminated_length": 13383.0, |
| "completions/mean_length": 8786.125, |
| "completions/mean_terminated_length": 8786.125, |
| "completions/min_length": 4108.0, |
| "completions/min_terminated_length": 4108.0, |
| "entropy": 0.30553513020277023, |
| "epoch": 0.2424969987995198, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4264076352119446, |
| "learning_rate": 1.96e-07, |
| "loss": 0.0064, |
| "num_tokens": 69208147.0, |
| "reward": 1.068824291229248, |
| "reward_std": 0.6439973711967468, |
| "rewards/reward_func/mean": 1.068824291229248, |
| "rewards/reward_func/std": 0.643997311592102, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9403151273727417, |
| "sampling/importance_sampling_ratio/mean": 0.9893855452537537, |
| "sampling/importance_sampling_ratio/min": 0.1719096601009369, |
| "sampling/sampling_logp_difference/max": 1.7607861757278442, |
| "sampling/sampling_logp_difference/mean": 0.02125393971800804, |
| "step": 202, |
| "step_time": 202.14960491785314 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13249.0, |
| "completions/max_terminated_length": 13249.0, |
| "completions/mean_length": 8494.1875, |
| "completions/mean_terminated_length": 8494.1875, |
| "completions/min_length": 2688.0, |
| "completions/min_terminated_length": 2688.0, |
| "entropy": 0.2879168214276433, |
| "epoch": 0.24369747899159663, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39069902896881104, |
| "learning_rate": 1.92e-07, |
| "loss": 0.0649, |
| "num_tokens": 69509457.0, |
| "reward": 1.7704880237579346, |
| "reward_std": 2.810868978500366, |
| "rewards/reward_func/mean": 1.7704880237579346, |
| "rewards/reward_func/std": 2.810868978500366, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9898999333381653, |
| "sampling/importance_sampling_ratio/min": 0.003532076720148325, |
| "sampling/sampling_logp_difference/max": 5.645869255065918, |
| "sampling/sampling_logp_difference/mean": 0.020249199122190475, |
| "step": 203, |
| "step_time": 183.80464980110992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13377.0, |
| "completions/max_terminated_length": 13377.0, |
| "completions/mean_length": 9125.28125, |
| "completions/mean_terminated_length": 9125.28125, |
| "completions/min_length": 4439.0, |
| "completions/min_terminated_length": 4439.0, |
| "entropy": 0.24132216069847345, |
| "epoch": 0.24489795918367346, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39876803755760193, |
| "learning_rate": 1.88e-07, |
| "loss": 0.0231, |
| "num_tokens": 69851202.0, |
| "reward": 0.5941991209983826, |
| "reward_std": 0.7350529432296753, |
| "rewards/reward_func/mean": 0.5941991209983826, |
| "rewards/reward_func/std": 0.7350529432296753, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8217852115631104, |
| "sampling/importance_sampling_ratio/mean": 0.991719663143158, |
| "sampling/importance_sampling_ratio/min": 0.302060604095459, |
| "sampling/sampling_logp_difference/max": 1.1971275806427002, |
| "sampling/sampling_logp_difference/mean": 0.016849037259817123, |
| "step": 204, |
| "step_time": 229.6054396291729 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12452.0, |
| "completions/max_terminated_length": 12452.0, |
| "completions/mean_length": 8173.46875, |
| "completions/mean_terminated_length": 8173.46875, |
| "completions/min_length": 4740.0, |
| "completions/min_terminated_length": 4740.0, |
| "entropy": 0.2661965722218156, |
| "epoch": 0.2460984393757503, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3894799053668976, |
| "learning_rate": 1.8399999999999998e-07, |
| "loss": -0.0405, |
| "num_tokens": 70146561.0, |
| "reward": 8.144660949707031, |
| "reward_std": 13.748440742492676, |
| "rewards/reward_func/mean": 8.144660949707031, |
| "rewards/reward_func/std": 13.748440742492676, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9909324049949646, |
| "sampling/importance_sampling_ratio/min": 0.17160779237747192, |
| "sampling/sampling_logp_difference/max": 1.7625436782836914, |
| "sampling/sampling_logp_difference/mean": 0.018414584919810295, |
| "step": 205, |
| "step_time": 194.27041996375192 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 20480.0, |
| "completions/max_terminated_length": 14681.0, |
| "completions/mean_length": 9778.9375, |
| "completions/mean_terminated_length": 9433.7412109375, |
| "completions/min_length": 5841.0, |
| "completions/min_terminated_length": 5841.0, |
| "entropy": 0.2878849096596241, |
| "epoch": 0.24729891956782712, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.23984801769256592, |
| "learning_rate": 1.8e-07, |
| "loss": -0.0947, |
| "num_tokens": 70496111.0, |
| "reward": 0.5641666650772095, |
| "reward_std": 0.7118456363677979, |
| "rewards/reward_func/mean": 0.6146236658096313, |
| "rewards/reward_func/std": 0.6628953814506531, |
| "rewards/soft_overlong_punishment_reward/mean": -0.03125, |
| "rewards/soft_overlong_punishment_reward/std": 0.1767766922712326, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989820122718811, |
| "sampling/importance_sampling_ratio/min": 0.3761327564716339, |
| "sampling/sampling_logp_difference/max": 1.2785313129425049, |
| "sampling/sampling_logp_difference/mean": 0.020214585587382317, |
| "step": 206, |
| "step_time": 256.00012533087283 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12854.0, |
| "completions/max_terminated_length": 12854.0, |
| "completions/mean_length": 7986.3125, |
| "completions/mean_terminated_length": 7986.3125, |
| "completions/min_length": 5362.0, |
| "completions/min_terminated_length": 5362.0, |
| "entropy": 0.26751279924064875, |
| "epoch": 0.24849939975990396, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.44527071714401245, |
| "learning_rate": 1.76e-07, |
| "loss": -0.0467, |
| "num_tokens": 70784649.0, |
| "reward": 1.1738154888153076, |
| "reward_std": 0.8459872603416443, |
| "rewards/reward_func/mean": 1.1738154888153076, |
| "rewards/reward_func/std": 0.8459872603416443, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.8473902940750122, |
| "sampling/importance_sampling_ratio/mean": 0.9906308650970459, |
| "sampling/importance_sampling_ratio/min": 0.3182202875614166, |
| "sampling/sampling_logp_difference/max": 1.1450114250183105, |
| "sampling/sampling_logp_difference/mean": 0.018941214308142662, |
| "step": 207, |
| "step_time": 183.06572339008562 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17590.0, |
| "completions/max_terminated_length": 17590.0, |
| "completions/mean_length": 10755.09375, |
| "completions/mean_terminated_length": 10755.09375, |
| "completions/min_length": 7289.0, |
| "completions/min_terminated_length": 7289.0, |
| "entropy": 0.28774886950850487, |
| "epoch": 0.24969987995198079, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3857475817203522, |
| "learning_rate": 1.7199999999999998e-07, |
| "loss": 0.0468, |
| "num_tokens": 71168340.0, |
| "reward": 1.0103178024291992, |
| "reward_std": 1.2723941802978516, |
| "rewards/reward_func/mean": 1.0195188522338867, |
| "rewards/reward_func/std": 1.2721405029296875, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0092010498046875, |
| "rewards/soft_overlong_punishment_reward/std": 0.052048999816179276, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9900601506233215, |
| "sampling/importance_sampling_ratio/min": 0.051096703857183456, |
| "sampling/sampling_logp_difference/max": 2.9740352630615234, |
| "sampling/sampling_logp_difference/mean": 0.01990516111254692, |
| "step": 208, |
| "step_time": 276.0685674578417 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16527.0, |
| "completions/max_terminated_length": 16527.0, |
| "completions/mean_length": 10267.90625, |
| "completions/mean_terminated_length": 10267.90625, |
| "completions/min_length": 5650.0, |
| "completions/min_terminated_length": 5650.0, |
| "entropy": 0.2723705815151334, |
| "epoch": 0.25090036014405764, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3713222146034241, |
| "learning_rate": 1.68e-07, |
| "loss": -0.014, |
| "num_tokens": 71532585.0, |
| "reward": 1.7478203773498535, |
| "reward_std": 2.0512068271636963, |
| "rewards/reward_func/mean": 1.7489113807678223, |
| "rewards/reward_func/std": 2.0506913661956787, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00109100341796875, |
| "rewards/soft_overlong_punishment_reward/std": 0.006171646993607283, |
| "sampling/importance_sampling_ratio/max": 2.043245553970337, |
| "sampling/importance_sampling_ratio/mean": 0.9906348586082458, |
| "sampling/importance_sampling_ratio/min": 0.11972002685070038, |
| "sampling/sampling_logp_difference/max": 2.1225993633270264, |
| "sampling/sampling_logp_difference/mean": 0.019026953727006912, |
| "step": 209, |
| "step_time": 249.78715924406424 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14411.0, |
| "completions/max_terminated_length": 14411.0, |
| "completions/mean_length": 8568.375, |
| "completions/mean_terminated_length": 8568.375, |
| "completions/min_length": 4104.0, |
| "completions/min_terminated_length": 4104.0, |
| "entropy": 0.3192238248884678, |
| "epoch": 0.25210084033613445, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4139728844165802, |
| "learning_rate": 1.64e-07, |
| "loss": -0.0156, |
| "num_tokens": 71830213.0, |
| "reward": 0.5381001830101013, |
| "reward_std": 0.655325710773468, |
| "rewards/reward_func/mean": 0.5381001830101013, |
| "rewards/reward_func/std": 0.655325710773468, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4862608909606934, |
| "sampling/importance_sampling_ratio/mean": 0.9889300465583801, |
| "sampling/importance_sampling_ratio/min": 0.2908362150192261, |
| "sampling/sampling_logp_difference/max": 1.234994888305664, |
| "sampling/sampling_logp_difference/mean": 0.021744508296251297, |
| "step": 210, |
| "step_time": 233.44613694911823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15175.0, |
| "completions/max_terminated_length": 15175.0, |
| "completions/mean_length": 6835.25, |
| "completions/mean_terminated_length": 6835.25, |
| "completions/min_length": 2738.0, |
| "completions/min_terminated_length": 2738.0, |
| "entropy": 0.2472194992005825, |
| "epoch": 0.2533013205282113, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3917876183986664, |
| "learning_rate": 1.6e-07, |
| "loss": 0.0568, |
| "num_tokens": 72084285.0, |
| "reward": 2.039097785949707, |
| "reward_std": 2.948822498321533, |
| "rewards/reward_func/mean": 2.039097785949707, |
| "rewards/reward_func/std": 2.9488227367401123, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.354682207107544, |
| "sampling/importance_sampling_ratio/mean": 0.991452157497406, |
| "sampling/importance_sampling_ratio/min": 0.037272509187459946, |
| "sampling/sampling_logp_difference/max": 3.289499282836914, |
| "sampling/sampling_logp_difference/mean": 0.017462395131587982, |
| "step": 211, |
| "step_time": 213.77216677484103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13379.0, |
| "completions/max_terminated_length": 13379.0, |
| "completions/mean_length": 8146.84375, |
| "completions/mean_terminated_length": 8146.84375, |
| "completions/min_length": 4187.0, |
| "completions/min_terminated_length": 4187.0, |
| "entropy": 0.27302129566669464, |
| "epoch": 0.2545018007202881, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3473767638206482, |
| "learning_rate": 1.56e-07, |
| "loss": 0.0159, |
| "num_tokens": 72380368.0, |
| "reward": 2.9426181316375732, |
| "reward_std": 3.704005718231201, |
| "rewards/reward_func/mean": 2.9426181316375732, |
| "rewards/reward_func/std": 3.704005479812622, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.7764674425125122, |
| "sampling/importance_sampling_ratio/mean": 0.9906426668167114, |
| "sampling/importance_sampling_ratio/min": 0.03155454993247986, |
| "sampling/sampling_logp_difference/max": 3.4560375213623047, |
| "sampling/sampling_logp_difference/mean": 0.01900378428399563, |
| "step": 212, |
| "step_time": 210.75419173389673 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14846.0, |
| "completions/max_terminated_length": 14846.0, |
| "completions/mean_length": 11225.4375, |
| "completions/mean_terminated_length": 11225.4375, |
| "completions/min_length": 7437.0, |
| "completions/min_terminated_length": 7437.0, |
| "entropy": 0.26801333110779524, |
| "epoch": 0.25570228091236497, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34166890382766724, |
| "learning_rate": 1.5199999999999998e-07, |
| "loss": -0.0198, |
| "num_tokens": 72773310.0, |
| "reward": 0.42110854387283325, |
| "reward_std": 0.5873985290527344, |
| "rewards/reward_func/mean": 0.42110854387283325, |
| "rewards/reward_func/std": 0.5873985290527344, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9907779693603516, |
| "sampling/importance_sampling_ratio/min": 0.34792667627334595, |
| "sampling/sampling_logp_difference/max": 1.1829512119293213, |
| "sampling/sampling_logp_difference/mean": 0.018622785806655884, |
| "step": 213, |
| "step_time": 239.31595174130052 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11657.0, |
| "completions/max_terminated_length": 11657.0, |
| "completions/mean_length": 7578.28125, |
| "completions/mean_terminated_length": 7578.28125, |
| "completions/min_length": 2938.0, |
| "completions/min_terminated_length": 2938.0, |
| "entropy": 0.28475786093622446, |
| "epoch": 0.25690276110444177, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4476838707923889, |
| "learning_rate": 1.4799999999999998e-07, |
| "loss": -0.0071, |
| "num_tokens": 73049535.0, |
| "reward": 1.1447203159332275, |
| "reward_std": 1.569128155708313, |
| "rewards/reward_func/mean": 1.1447203159332275, |
| "rewards/reward_func/std": 1.569128155708313, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.6962001323699951, |
| "sampling/importance_sampling_ratio/mean": 0.9899388551712036, |
| "sampling/importance_sampling_ratio/min": 0.2515447735786438, |
| "sampling/sampling_logp_difference/max": 1.3801343441009521, |
| "sampling/sampling_logp_difference/mean": 0.019816022366285324, |
| "step": 214, |
| "step_time": 192.09097037103493 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15818.0, |
| "completions/max_terminated_length": 15818.0, |
| "completions/mean_length": 9813.6875, |
| "completions/mean_terminated_length": 9813.6875, |
| "completions/min_length": 5428.0, |
| "completions/min_terminated_length": 5428.0, |
| "entropy": 0.2705380003899336, |
| "epoch": 0.2581032412965186, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38898560404777527, |
| "learning_rate": 1.44e-07, |
| "loss": 0.0382, |
| "num_tokens": 73403117.0, |
| "reward": 0.6952791810035706, |
| "reward_std": 0.6618348360061646, |
| "rewards/reward_func/mean": 0.6952791810035706, |
| "rewards/reward_func/std": 0.6618348360061646, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8969855308532715, |
| "sampling/importance_sampling_ratio/mean": 0.9905439615249634, |
| "sampling/importance_sampling_ratio/min": 0.05445502698421478, |
| "sampling/sampling_logp_difference/max": 2.9103801250457764, |
| "sampling/sampling_logp_difference/mean": 0.018809445202350616, |
| "step": 215, |
| "step_time": 219.26422131690197 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12131.0, |
| "completions/max_terminated_length": 12131.0, |
| "completions/mean_length": 9191.46875, |
| "completions/mean_terminated_length": 9191.46875, |
| "completions/min_length": 4309.0, |
| "completions/min_terminated_length": 4309.0, |
| "entropy": 0.29348371736705303, |
| "epoch": 0.25930372148859543, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4402393102645874, |
| "learning_rate": 1.4e-07, |
| "loss": 0.0291, |
| "num_tokens": 73730012.0, |
| "reward": 2.512660264968872, |
| "reward_std": 2.8215131759643555, |
| "rewards/reward_func/mean": 2.512660264968872, |
| "rewards/reward_func/std": 2.8215129375457764, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989812970161438, |
| "sampling/importance_sampling_ratio/min": 0.16945770382881165, |
| "sampling/sampling_logp_difference/max": 1.7751518487930298, |
| "sampling/sampling_logp_difference/mean": 0.020123887807130814, |
| "step": 216, |
| "step_time": 192.42200359026901 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14410.0, |
| "completions/max_terminated_length": 14410.0, |
| "completions/mean_length": 10286.53125, |
| "completions/mean_terminated_length": 10286.53125, |
| "completions/min_length": 6044.0, |
| "completions/min_terminated_length": 6044.0, |
| "entropy": 0.2660679593682289, |
| "epoch": 0.2605042016806723, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3704564571380615, |
| "learning_rate": 1.36e-07, |
| "loss": 0.0008, |
| "num_tokens": 74100413.0, |
| "reward": 0.5737283229827881, |
| "reward_std": 0.6282573342323303, |
| "rewards/reward_func/mean": 0.5737283229827881, |
| "rewards/reward_func/std": 0.6282573342323303, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.249643325805664, |
| "sampling/importance_sampling_ratio/mean": 0.9908007979393005, |
| "sampling/importance_sampling_ratio/min": 0.00029296314460225403, |
| "sampling/sampling_logp_difference/max": 8.13546371459961, |
| "sampling/sampling_logp_difference/mean": 0.018784763291478157, |
| "step": 217, |
| "step_time": 440.4557863563532 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14001.0, |
| "completions/max_terminated_length": 14001.0, |
| "completions/mean_length": 10060.25, |
| "completions/mean_terminated_length": 10060.25, |
| "completions/min_length": 6824.0, |
| "completions/min_terminated_length": 6824.0, |
| "entropy": 0.3022372853010893, |
| "epoch": 0.2617046818727491, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43306854367256165, |
| "learning_rate": 1.32e-07, |
| "loss": 0.0028, |
| "num_tokens": 74455253.0, |
| "reward": 16.12359619140625, |
| "reward_std": 59.91655731201172, |
| "rewards/reward_func/mean": 16.12359619140625, |
| "rewards/reward_func/std": 59.91655349731445, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.2559473514556885, |
| "sampling/importance_sampling_ratio/mean": 0.9893619418144226, |
| "sampling/importance_sampling_ratio/min": 0.22430028021335602, |
| "sampling/sampling_logp_difference/max": 1.49476957321167, |
| "sampling/sampling_logp_difference/mean": 0.021203139796853065, |
| "step": 218, |
| "step_time": 414.3311550068902 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11622.0, |
| "completions/max_terminated_length": 11622.0, |
| "completions/mean_length": 6517.96875, |
| "completions/mean_terminated_length": 6517.96875, |
| "completions/min_length": 3112.0, |
| "completions/min_terminated_length": 3112.0, |
| "entropy": 0.28310599736869335, |
| "epoch": 0.26290516206482595, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.447173535823822, |
| "learning_rate": 1.28e-07, |
| "loss": -0.0289, |
| "num_tokens": 74686092.0, |
| "reward": 0.6452304720878601, |
| "reward_std": 2.197897434234619, |
| "rewards/reward_func/mean": 0.6452304720878601, |
| "rewards/reward_func/std": 2.197897434234619, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.134779691696167, |
| "sampling/importance_sampling_ratio/mean": 0.9900719523429871, |
| "sampling/importance_sampling_ratio/min": 0.36751604080200195, |
| "sampling/sampling_logp_difference/max": 1.0009883642196655, |
| "sampling/sampling_logp_difference/mean": 0.0198367852717638, |
| "step": 219, |
| "step_time": 177.24385808466468 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11380.0, |
| "completions/max_terminated_length": 11380.0, |
| "completions/mean_length": 6471.75, |
| "completions/mean_terminated_length": 6471.75, |
| "completions/min_length": 4281.0, |
| "completions/min_terminated_length": 4281.0, |
| "entropy": 0.3177992310374975, |
| "epoch": 0.26410564225690275, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5168129205703735, |
| "learning_rate": 1.24e-07, |
| "loss": 0.0395, |
| "num_tokens": 74912764.0, |
| "reward": 2.1183838844299316, |
| "reward_std": 1.930953025817871, |
| "rewards/reward_func/mean": 2.1183838844299316, |
| "rewards/reward_func/std": 1.930953025817871, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0708396434783936, |
| "sampling/importance_sampling_ratio/mean": 0.9886492490768433, |
| "sampling/importance_sampling_ratio/min": 0.3355446755886078, |
| "sampling/sampling_logp_difference/max": 1.0920002460479736, |
| "sampling/sampling_logp_difference/mean": 0.022046584635972977, |
| "step": 220, |
| "step_time": 158.16589547123294 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14608.0, |
| "completions/max_terminated_length": 14608.0, |
| "completions/mean_length": 9121.625, |
| "completions/mean_terminated_length": 9121.625, |
| "completions/min_length": 2448.0, |
| "completions/min_terminated_length": 2448.0, |
| "entropy": 0.2602370521053672, |
| "epoch": 0.2653061224489796, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39733272790908813, |
| "learning_rate": 1.2e-07, |
| "loss": 0.0234, |
| "num_tokens": 75242344.0, |
| "reward": 0.6069552898406982, |
| "reward_std": 0.6848835945129395, |
| "rewards/reward_func/mean": 0.6069552898406982, |
| "rewards/reward_func/std": 0.6848835349082947, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.775014877319336, |
| "sampling/importance_sampling_ratio/mean": 0.9909803867340088, |
| "sampling/importance_sampling_ratio/min": 0.14724299311637878, |
| "sampling/sampling_logp_difference/max": 1.9156711101531982, |
| "sampling/sampling_logp_difference/mean": 0.01835060492157936, |
| "step": 221, |
| "step_time": 228.61106476001441 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12311.0, |
| "completions/max_terminated_length": 12311.0, |
| "completions/mean_length": 8921.34375, |
| "completions/mean_terminated_length": 8921.34375, |
| "completions/min_length": 5336.0, |
| "completions/min_terminated_length": 5336.0, |
| "entropy": 0.2625180771574378, |
| "epoch": 0.2665066026410564, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3746626079082489, |
| "learning_rate": 1.16e-07, |
| "loss": -0.0205, |
| "num_tokens": 75562803.0, |
| "reward": 1.2791639566421509, |
| "reward_std": 1.4882169961929321, |
| "rewards/reward_func/mean": 1.2791639566421509, |
| "rewards/reward_func/std": 1.4882171154022217, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9303572177886963, |
| "sampling/importance_sampling_ratio/mean": 0.9908260107040405, |
| "sampling/importance_sampling_ratio/min": 0.32418113946914673, |
| "sampling/sampling_logp_difference/max": 1.1264528036117554, |
| "sampling/sampling_logp_difference/mean": 0.01839890144765377, |
| "step": 222, |
| "step_time": 331.951701607788 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15993.0, |
| "completions/max_terminated_length": 15993.0, |
| "completions/mean_length": 9919.09375, |
| "completions/mean_terminated_length": 9919.09375, |
| "completions/min_length": 2850.0, |
| "completions/min_terminated_length": 2850.0, |
| "entropy": 0.27178078424185514, |
| "epoch": 0.26770708283313327, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.32960885763168335, |
| "learning_rate": 1.12e-07, |
| "loss": -0.0391, |
| "num_tokens": 75917510.0, |
| "reward": 0.9057698845863342, |
| "reward_std": 1.4171531200408936, |
| "rewards/reward_func/mean": 0.9057698845863342, |
| "rewards/reward_func/std": 1.4171531200408936, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9905833005905151, |
| "sampling/importance_sampling_ratio/min": 0.12444481998682022, |
| "sampling/sampling_logp_difference/max": 2.083892822265625, |
| "sampling/sampling_logp_difference/mean": 0.019010471180081367, |
| "step": 223, |
| "step_time": 452.7820453970926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15207.0, |
| "completions/max_terminated_length": 15207.0, |
| "completions/mean_length": 8989.75, |
| "completions/mean_terminated_length": 8989.75, |
| "completions/min_length": 5085.0, |
| "completions/min_terminated_length": 5085.0, |
| "entropy": 0.27119907177984715, |
| "epoch": 0.2689075630252101, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3286755681037903, |
| "learning_rate": 1.0799999999999999e-07, |
| "loss": -0.01, |
| "num_tokens": 76241470.0, |
| "reward": 2.0997040271759033, |
| "reward_std": 2.725891351699829, |
| "rewards/reward_func/mean": 2.0997040271759033, |
| "rewards/reward_func/std": 2.72589111328125, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.770100474357605, |
| "sampling/importance_sampling_ratio/mean": 0.9906846284866333, |
| "sampling/importance_sampling_ratio/min": 0.03444494679570198, |
| "sampling/sampling_logp_difference/max": 3.3683929443359375, |
| "sampling/sampling_logp_difference/mean": 0.018905045464634895, |
| "step": 224, |
| "step_time": 434.58480245189276 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14341.0, |
| "completions/max_terminated_length": 14341.0, |
| "completions/mean_length": 9169.65625, |
| "completions/mean_terminated_length": 9169.65625, |
| "completions/min_length": 4655.0, |
| "completions/min_terminated_length": 4655.0, |
| "entropy": 0.2545336727052927, |
| "epoch": 0.27010804321728693, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43626582622528076, |
| "learning_rate": 1.0399999999999999e-07, |
| "loss": -0.0511, |
| "num_tokens": 76579227.0, |
| "reward": 0.7725225687026978, |
| "reward_std": 0.8143731951713562, |
| "rewards/reward_func/mean": 0.7725225687026978, |
| "rewards/reward_func/std": 0.8143731951713562, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.39496111869812, |
| "sampling/importance_sampling_ratio/mean": 0.9909839630126953, |
| "sampling/importance_sampling_ratio/min": 0.17538554966449738, |
| "sampling/sampling_logp_difference/max": 1.7407686710357666, |
| "sampling/sampling_logp_difference/mean": 0.018118245527148247, |
| "step": 225, |
| "step_time": 256.96223465609364 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11969.0, |
| "completions/max_terminated_length": 11969.0, |
| "completions/mean_length": 7255.71875, |
| "completions/mean_terminated_length": 7255.71875, |
| "completions/min_length": 3750.0, |
| "completions/min_terminated_length": 3750.0, |
| "entropy": 0.2862818920984864, |
| "epoch": 0.27130852340936373, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5091444253921509, |
| "learning_rate": 1e-07, |
| "loss": 0.0555, |
| "num_tokens": 76838802.0, |
| "reward": 3.338810443878174, |
| "reward_std": 3.9317619800567627, |
| "rewards/reward_func/mean": 3.338810443878174, |
| "rewards/reward_func/std": 3.9317619800567627, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1975717544555664, |
| "sampling/importance_sampling_ratio/mean": 0.9902316331863403, |
| "sampling/importance_sampling_ratio/min": 0.10265523940324783, |
| "sampling/sampling_logp_difference/max": 2.276379108428955, |
| "sampling/sampling_logp_difference/mean": 0.01985924504697323, |
| "step": 226, |
| "step_time": 177.48551015171688 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14337.0, |
| "completions/max_terminated_length": 14337.0, |
| "completions/mean_length": 8913.25, |
| "completions/mean_terminated_length": 8913.25, |
| "completions/min_length": 3138.0, |
| "completions/min_terminated_length": 3138.0, |
| "entropy": 0.26585716381669044, |
| "epoch": 0.2725090036014406, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3888947069644928, |
| "learning_rate": 9.6e-08, |
| "loss": -0.0538, |
| "num_tokens": 77161562.0, |
| "reward": 2.5214858055114746, |
| "reward_std": 2.8809354305267334, |
| "rewards/reward_func/mean": 2.5214858055114746, |
| "rewards/reward_func/std": 2.8809354305267334, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.9848451614379883, |
| "sampling/importance_sampling_ratio/mean": 0.9906054139137268, |
| "sampling/importance_sampling_ratio/min": 0.12535132467746735, |
| "sampling/sampling_logp_difference/max": 2.0766348838806152, |
| "sampling/sampling_logp_difference/mean": 0.01874399371445179, |
| "step": 227, |
| "step_time": 423.3902858378133 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12350.0, |
| "completions/max_terminated_length": 12350.0, |
| "completions/mean_length": 9451.5625, |
| "completions/mean_terminated_length": 9451.5625, |
| "completions/min_length": 6397.0, |
| "completions/min_terminated_length": 6397.0, |
| "entropy": 0.2693768171593547, |
| "epoch": 0.2737094837935174, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3820766508579254, |
| "learning_rate": 9.199999999999999e-08, |
| "loss": 0.033, |
| "num_tokens": 77498220.0, |
| "reward": 1.154233455657959, |
| "reward_std": 1.959659218788147, |
| "rewards/reward_func/mean": 1.154233455657959, |
| "rewards/reward_func/std": 1.959659218788147, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9909024238586426, |
| "sampling/importance_sampling_ratio/min": 0.20406676828861237, |
| "sampling/sampling_logp_difference/max": 1.5893080234527588, |
| "sampling/sampling_logp_difference/mean": 0.018469782546162605, |
| "step": 228, |
| "step_time": 217.56979901902378 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12107.0, |
| "completions/max_terminated_length": 12107.0, |
| "completions/mean_length": 8378.96875, |
| "completions/mean_terminated_length": 8378.96875, |
| "completions/min_length": 5003.0, |
| "completions/min_terminated_length": 5003.0, |
| "entropy": 0.2748628966510296, |
| "epoch": 0.27490996398559425, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4501209557056427, |
| "learning_rate": 8.8e-08, |
| "loss": -0.0042, |
| "num_tokens": 77802059.0, |
| "reward": 1.3091723918914795, |
| "reward_std": 0.47337406873703003, |
| "rewards/reward_func/mean": 1.3091723918914795, |
| "rewards/reward_func/std": 0.47337406873703003, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.0756125450134277, |
| "sampling/importance_sampling_ratio/mean": 0.9905352592468262, |
| "sampling/importance_sampling_ratio/min": 0.22848102450370789, |
| "sampling/sampling_logp_difference/max": 1.476302146911621, |
| "sampling/sampling_logp_difference/mean": 0.01908484473824501, |
| "step": 229, |
| "step_time": 173.2410452858312 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11253.0, |
| "completions/max_terminated_length": 11253.0, |
| "completions/mean_length": 8427.15625, |
| "completions/mean_terminated_length": 8427.15625, |
| "completions/min_length": 5975.0, |
| "completions/min_terminated_length": 5975.0, |
| "entropy": 0.28478993009775877, |
| "epoch": 0.27611044417767105, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42257431149482727, |
| "learning_rate": 8.4e-08, |
| "loss": 0.0338, |
| "num_tokens": 78102984.0, |
| "reward": 0.6847608089447021, |
| "reward_std": 0.6159871220588684, |
| "rewards/reward_func/mean": 0.6847608089447021, |
| "rewards/reward_func/std": 0.6159871220588684, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.810465693473816, |
| "sampling/importance_sampling_ratio/mean": 0.9901672601699829, |
| "sampling/importance_sampling_ratio/min": 0.21681459248065948, |
| "sampling/sampling_logp_difference/max": 1.5287127494812012, |
| "sampling/sampling_logp_difference/mean": 0.01989746280014515, |
| "step": 230, |
| "step_time": 376.1443422808079 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13502.0, |
| "completions/max_terminated_length": 13502.0, |
| "completions/mean_length": 9104.8125, |
| "completions/mean_terminated_length": 9104.8125, |
| "completions/min_length": 5037.0, |
| "completions/min_terminated_length": 5037.0, |
| "entropy": 0.28373472206294537, |
| "epoch": 0.2773109243697479, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3297131657600403, |
| "learning_rate": 8e-08, |
| "loss": -0.004, |
| "num_tokens": 78423906.0, |
| "reward": 3.833698272705078, |
| "reward_std": 8.968595504760742, |
| "rewards/reward_func/mean": 3.833698272705078, |
| "rewards/reward_func/std": 8.968595504760742, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.7473583221435547, |
| "sampling/importance_sampling_ratio/mean": 0.9901688694953918, |
| "sampling/importance_sampling_ratio/min": 0.3091135621070862, |
| "sampling/sampling_logp_difference/max": 1.174046516418457, |
| "sampling/sampling_logp_difference/mean": 0.019583595916628838, |
| "step": 231, |
| "step_time": 192.75546575232875 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13344.0, |
| "completions/max_terminated_length": 13344.0, |
| "completions/mean_length": 9607.15625, |
| "completions/mean_terminated_length": 9607.15625, |
| "completions/min_length": 6523.0, |
| "completions/min_terminated_length": 6523.0, |
| "entropy": 0.258233230561018, |
| "epoch": 0.2785114045618247, |
| "frac_reward_zero_std": 0.75, |
| "grad_norm": 0.22545225918293, |
| "learning_rate": 7.599999999999999e-08, |
| "loss": 0.0098, |
| "num_tokens": 78774783.0, |
| "reward": 1.9562370777130127, |
| "reward_std": 3.413482666015625, |
| "rewards/reward_func/mean": 1.9562370777130127, |
| "rewards/reward_func/std": 3.413482666015625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9909834861755371, |
| "sampling/importance_sampling_ratio/min": 0.03608707711100578, |
| "sampling/sampling_logp_difference/max": 3.3218204975128174, |
| "sampling/sampling_logp_difference/mean": 0.01814298704266548, |
| "step": 232, |
| "step_time": 441.14181373117026 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14557.0, |
| "completions/max_terminated_length": 14557.0, |
| "completions/mean_length": 8880.46875, |
| "completions/mean_terminated_length": 8880.46875, |
| "completions/min_length": 3949.0, |
| "completions/min_terminated_length": 3949.0, |
| "entropy": 0.2796018850058317, |
| "epoch": 0.2797118847539016, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41359373927116394, |
| "learning_rate": 7.2e-08, |
| "loss": 0.0204, |
| "num_tokens": 79094742.0, |
| "reward": 2.0865750312805176, |
| "reward_std": 3.248898506164551, |
| "rewards/reward_func/mean": 2.0865750312805176, |
| "rewards/reward_func/std": 3.248898506164551, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.355583906173706, |
| "sampling/importance_sampling_ratio/mean": 0.9904192090034485, |
| "sampling/importance_sampling_ratio/min": 0.22791339457035065, |
| "sampling/sampling_logp_difference/max": 1.4787895679473877, |
| "sampling/sampling_logp_difference/mean": 0.019396543502807617, |
| "step": 233, |
| "step_time": 433.1994162177434 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13023.0, |
| "completions/max_terminated_length": 13023.0, |
| "completions/mean_length": 9278.125, |
| "completions/mean_terminated_length": 9278.125, |
| "completions/min_length": 6046.0, |
| "completions/min_terminated_length": 6046.0, |
| "entropy": 0.23018638603389263, |
| "epoch": 0.2809123649459784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40194013714790344, |
| "learning_rate": 6.8e-08, |
| "loss": -0.0207, |
| "num_tokens": 79432034.0, |
| "reward": 1.2649976015090942, |
| "reward_std": 2.4887611865997314, |
| "rewards/reward_func/mean": 1.2649976015090942, |
| "rewards/reward_func/std": 2.4887611865997314, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.066953420639038, |
| "sampling/importance_sampling_ratio/mean": 0.9919387698173523, |
| "sampling/importance_sampling_ratio/min": 0.07596759498119354, |
| "sampling/sampling_logp_difference/max": 2.5774483680725098, |
| "sampling/sampling_logp_difference/mean": 0.01637454889714718, |
| "step": 234, |
| "step_time": 200.44299350422807 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11484.0, |
| "completions/max_terminated_length": 11484.0, |
| "completions/mean_length": 7951.46875, |
| "completions/mean_terminated_length": 7951.46875, |
| "completions/min_length": 4951.0, |
| "completions/min_terminated_length": 4951.0, |
| "entropy": 0.28384537622332573, |
| "epoch": 0.28211284513805523, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4639303684234619, |
| "learning_rate": 6.4e-08, |
| "loss": 0.0062, |
| "num_tokens": 79715057.0, |
| "reward": 1.722654938697815, |
| "reward_std": 2.3874762058258057, |
| "rewards/reward_func/mean": 1.722654938697815, |
| "rewards/reward_func/std": 2.3874759674072266, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9900355339050293, |
| "sampling/importance_sampling_ratio/min": 0.16387537121772766, |
| "sampling/sampling_logp_difference/max": 1.8086490631103516, |
| "sampling/sampling_logp_difference/mean": 0.019974499940872192, |
| "step": 235, |
| "step_time": 188.83989403117448 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15359.0, |
| "completions/max_terminated_length": 15359.0, |
| "completions/mean_length": 9800.75, |
| "completions/mean_terminated_length": 9800.75, |
| "completions/min_length": 6034.0, |
| "completions/min_terminated_length": 6034.0, |
| "entropy": 0.3060651607811451, |
| "epoch": 0.28331332533013204, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4038175940513611, |
| "learning_rate": 6e-08, |
| "loss": 0.0295, |
| "num_tokens": 80063393.0, |
| "reward": 0.5499008893966675, |
| "reward_std": 0.9594078660011292, |
| "rewards/reward_func/mean": 0.5499008893966675, |
| "rewards/reward_func/std": 0.9594078660011292, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.989382803440094, |
| "sampling/importance_sampling_ratio/min": 0.10317613184452057, |
| "sampling/sampling_logp_difference/max": 2.271317720413208, |
| "sampling/sampling_logp_difference/mean": 0.021366208791732788, |
| "step": 236, |
| "step_time": 246.472208352061 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14811.0, |
| "completions/max_terminated_length": 14811.0, |
| "completions/mean_length": 10420.625, |
| "completions/mean_terminated_length": 10420.625, |
| "completions/min_length": 7528.0, |
| "completions/min_terminated_length": 7528.0, |
| "entropy": 0.26841907389461994, |
| "epoch": 0.2845138055222089, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3900371789932251, |
| "learning_rate": 5.6e-08, |
| "loss": 0.0085, |
| "num_tokens": 80437933.0, |
| "reward": 0.7362344264984131, |
| "reward_std": 0.9423819780349731, |
| "rewards/reward_func/mean": 0.7362344264984131, |
| "rewards/reward_func/std": 0.9423819184303284, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9905475378036499, |
| "sampling/importance_sampling_ratio/min": 0.06430970132350922, |
| "sampling/sampling_logp_difference/max": 2.744044780731201, |
| "sampling/sampling_logp_difference/mean": 0.018762534484267235, |
| "step": 237, |
| "step_time": 224.29740512394346 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19168.0, |
| "completions/max_terminated_length": 19168.0, |
| "completions/mean_length": 10909.96875, |
| "completions/mean_terminated_length": 10909.96875, |
| "completions/min_length": 7484.0, |
| "completions/min_terminated_length": 7484.0, |
| "entropy": 0.2649214630946517, |
| "epoch": 0.2857142857142857, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3601086735725403, |
| "learning_rate": 5.1999999999999996e-08, |
| "loss": 0.0136, |
| "num_tokens": 80832676.0, |
| "reward": 1.2694827318191528, |
| "reward_std": 1.707560420036316, |
| "rewards/reward_func/mean": 1.2907229661941528, |
| "rewards/reward_func/std": 1.7039649486541748, |
| "rewards/soft_overlong_punishment_reward/mean": -0.021240234375, |
| "rewards/soft_overlong_punishment_reward/std": 0.12015291303396225, |
| "sampling/importance_sampling_ratio/max": 2.5833005905151367, |
| "sampling/importance_sampling_ratio/mean": 0.990875244140625, |
| "sampling/importance_sampling_ratio/min": 0.3051411211490631, |
| "sampling/sampling_logp_difference/max": 1.186980962753296, |
| "sampling/sampling_logp_difference/mean": 0.018429221585392952, |
| "step": 238, |
| "step_time": 254.5579416519031 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13040.0, |
| "completions/max_terminated_length": 13040.0, |
| "completions/mean_length": 7543.53125, |
| "completions/mean_terminated_length": 7543.53125, |
| "completions/min_length": 4494.0, |
| "completions/min_terminated_length": 4494.0, |
| "entropy": 0.2839345559477806, |
| "epoch": 0.28691476590636256, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.487249493598938, |
| "learning_rate": 4.8e-08, |
| "loss": 0.028, |
| "num_tokens": 81106021.0, |
| "reward": 0.7453573942184448, |
| "reward_std": 0.7949322462081909, |
| "rewards/reward_func/mean": 0.7453573942184448, |
| "rewards/reward_func/std": 0.7949321269989014, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.3466830253601074, |
| "sampling/importance_sampling_ratio/mean": 0.9900220036506653, |
| "sampling/importance_sampling_ratio/min": 0.012635735794901848, |
| "sampling/sampling_logp_difference/max": 4.3712263107299805, |
| "sampling/sampling_logp_difference/mean": 0.019755955785512924, |
| "step": 239, |
| "step_time": 204.47343361668754 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13324.0, |
| "completions/max_terminated_length": 13324.0, |
| "completions/mean_length": 8653.3125, |
| "completions/mean_terminated_length": 8653.3125, |
| "completions/min_length": 4673.0, |
| "completions/min_terminated_length": 4673.0, |
| "entropy": 0.2574413791298866, |
| "epoch": 0.28811524609843936, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3604678213596344, |
| "learning_rate": 4.4e-08, |
| "loss": -0.0105, |
| "num_tokens": 81421999.0, |
| "reward": 1.671258568763733, |
| "reward_std": 2.822765350341797, |
| "rewards/reward_func/mean": 1.671258568763733, |
| "rewards/reward_func/std": 2.822765588760376, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.407453775405884, |
| "sampling/importance_sampling_ratio/mean": 0.9911367893218994, |
| "sampling/importance_sampling_ratio/min": 0.35551854968070984, |
| "sampling/sampling_logp_difference/max": 1.0341778993606567, |
| "sampling/sampling_logp_difference/mean": 0.018147394061088562, |
| "step": 240, |
| "step_time": 216.4728114741156 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15334.0, |
| "completions/max_terminated_length": 15334.0, |
| "completions/mean_length": 10050.375, |
| "completions/mean_terminated_length": 10050.375, |
| "completions/min_length": 3177.0, |
| "completions/min_terminated_length": 3177.0, |
| "entropy": 0.25266273971647024, |
| "epoch": 0.2893157262905162, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.34388092160224915, |
| "learning_rate": 4e-08, |
| "loss": -0.0302, |
| "num_tokens": 81783875.0, |
| "reward": 2.149714469909668, |
| "reward_std": 2.923807144165039, |
| "rewards/reward_func/mean": 2.149714469909668, |
| "rewards/reward_func/std": 2.92380690574646, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9911876916885376, |
| "sampling/importance_sampling_ratio/min": 0.00033268670085817575, |
| "sampling/sampling_logp_difference/max": 8.008309364318848, |
| "sampling/sampling_logp_difference/mean": 0.01783919706940651, |
| "step": 241, |
| "step_time": 258.6670099237235 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10994.0, |
| "completions/max_terminated_length": 10994.0, |
| "completions/mean_length": 8268.15625, |
| "completions/mean_terminated_length": 8268.15625, |
| "completions/min_length": 5467.0, |
| "completions/min_terminated_length": 5467.0, |
| "entropy": 0.2947918279096484, |
| "epoch": 0.290516206482593, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.37617602944374084, |
| "learning_rate": 3.6e-08, |
| "loss": 0.045, |
| "num_tokens": 82076200.0, |
| "reward": 1.3230187892913818, |
| "reward_std": 1.936329960823059, |
| "rewards/reward_func/mean": 1.3230187892913818, |
| "rewards/reward_func/std": 1.9363298416137695, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.1945412158966064, |
| "sampling/importance_sampling_ratio/mean": 0.98971027135849, |
| "sampling/importance_sampling_ratio/min": 0.0968865156173706, |
| "sampling/sampling_logp_difference/max": 2.334214925765991, |
| "sampling/sampling_logp_difference/mean": 0.020341265946626663, |
| "step": 242, |
| "step_time": 192.7506662090309 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12898.0, |
| "completions/max_terminated_length": 12898.0, |
| "completions/mean_length": 9256.78125, |
| "completions/mean_terminated_length": 9256.78125, |
| "completions/min_length": 4686.0, |
| "completions/min_terminated_length": 4686.0, |
| "entropy": 0.30538202077150345, |
| "epoch": 0.2917166866746699, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.3372436761856079, |
| "learning_rate": 3.2e-08, |
| "loss": 0.0148, |
| "num_tokens": 82395961.0, |
| "reward": 0.8563218712806702, |
| "reward_std": 1.9926910400390625, |
| "rewards/reward_func/mean": 0.8563218712806702, |
| "rewards/reward_func/std": 1.9926910400390625, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9894178509712219, |
| "sampling/importance_sampling_ratio/min": 0.13205546140670776, |
| "sampling/sampling_logp_difference/max": 2.024533271789551, |
| "sampling/sampling_logp_difference/mean": 0.02108946070075035, |
| "step": 243, |
| "step_time": 222.2967141327681 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13215.0, |
| "completions/max_terminated_length": 13215.0, |
| "completions/mean_length": 9513.03125, |
| "completions/mean_terminated_length": 9513.03125, |
| "completions/min_length": 6086.0, |
| "completions/min_terminated_length": 6086.0, |
| "entropy": 0.26438521407544613, |
| "epoch": 0.2929171668667467, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40192142128944397, |
| "learning_rate": 2.8e-08, |
| "loss": 0.0154, |
| "num_tokens": 82738746.0, |
| "reward": 3.0766522884368896, |
| "reward_std": 2.7076849937438965, |
| "rewards/reward_func/mean": 3.0766522884368896, |
| "rewards/reward_func/std": 2.7076849937438965, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9906589388847351, |
| "sampling/importance_sampling_ratio/min": 0.27606433629989624, |
| "sampling/sampling_logp_difference/max": 1.287121295928955, |
| "sampling/sampling_logp_difference/mean": 0.018659161403775215, |
| "step": 244, |
| "step_time": 206.91322167252656 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 17517.0, |
| "completions/max_terminated_length": 17517.0, |
| "completions/mean_length": 9881.0625, |
| "completions/mean_terminated_length": 9881.0625, |
| "completions/min_length": 2381.0, |
| "completions/min_terminated_length": 2381.0, |
| "entropy": 0.25762535352259874, |
| "epoch": 0.29411764705882354, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40251660346984863, |
| "learning_rate": 2.4e-08, |
| "loss": 0.0311, |
| "num_tokens": 83097732.0, |
| "reward": 3.15523099899292, |
| "reward_std": 5.307023525238037, |
| "rewards/reward_func/mean": 3.163875102996826, |
| "rewards/reward_func/std": 5.301476001739502, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00864410400390625, |
| "rewards/soft_overlong_punishment_reward/std": 0.04889843612909317, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9913111925125122, |
| "sampling/importance_sampling_ratio/min": 0.0033161414321511984, |
| "sampling/sampling_logp_difference/max": 5.708953380584717, |
| "sampling/sampling_logp_difference/mean": 0.01810595765709877, |
| "step": 245, |
| "step_time": 277.0211471380899 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 16853.0, |
| "completions/max_terminated_length": 16853.0, |
| "completions/mean_length": 10218.5, |
| "completions/mean_terminated_length": 10218.5, |
| "completions/min_length": 6606.0, |
| "completions/min_terminated_length": 6606.0, |
| "entropy": 0.28045162558555603, |
| "epoch": 0.29531812725090034, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3734090030193329, |
| "learning_rate": 2e-08, |
| "loss": 0.001, |
| "num_tokens": 83459092.0, |
| "reward": 0.8230442404747009, |
| "reward_std": 0.6862863898277283, |
| "rewards/reward_func/mean": 0.8266224265098572, |
| "rewards/reward_func/std": 0.6817933917045593, |
| "rewards/soft_overlong_punishment_reward/mean": -0.00357818603515625, |
| "rewards/soft_overlong_punishment_reward/std": 0.020241277292370796, |
| "sampling/importance_sampling_ratio/max": 2.0942885875701904, |
| "sampling/importance_sampling_ratio/mean": 0.990305483341217, |
| "sampling/importance_sampling_ratio/min": 0.018712347373366356, |
| "sampling/sampling_logp_difference/max": 3.978571653366089, |
| "sampling/sampling_logp_difference/mean": 0.019516419619321823, |
| "step": 246, |
| "step_time": 472.73078032024205 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 19484.0, |
| "completions/max_terminated_length": 19484.0, |
| "completions/mean_length": 10279.5625, |
| "completions/mean_terminated_length": 10279.5625, |
| "completions/min_length": 5641.0, |
| "completions/min_terminated_length": 5641.0, |
| "entropy": 0.2904065754264593, |
| "epoch": 0.2965186074429772, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3862820565700531, |
| "learning_rate": 1.6e-08, |
| "loss": 0.032, |
| "num_tokens": 83814838.0, |
| "reward": 1.4464482069015503, |
| "reward_std": 2.287731885910034, |
| "rewards/reward_func/mean": 1.4815586805343628, |
| "rewards/reward_func/std": 2.2599167823791504, |
| "rewards/soft_overlong_punishment_reward/mean": -0.0351104736328125, |
| "rewards/soft_overlong_punishment_reward/std": 0.146774023771286, |
| "sampling/importance_sampling_ratio/max": 2.4637815952301025, |
| "sampling/importance_sampling_ratio/mean": 0.989745557308197, |
| "sampling/importance_sampling_ratio/min": 0.15747244656085968, |
| "sampling/sampling_logp_difference/max": 1.8485047817230225, |
| "sampling/sampling_logp_difference/mean": 0.020400259643793106, |
| "step": 247, |
| "step_time": 332.2391231590882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12802.0, |
| "completions/max_terminated_length": 12802.0, |
| "completions/mean_length": 7361.375, |
| "completions/mean_terminated_length": 7361.375, |
| "completions/min_length": 5259.0, |
| "completions/min_terminated_length": 5259.0, |
| "entropy": 0.3144523575901985, |
| "epoch": 0.297719087635054, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.45082876086235046, |
| "learning_rate": 1.2e-08, |
| "loss": 0.0151, |
| "num_tokens": 84073074.0, |
| "reward": 1.048694372177124, |
| "reward_std": 0.5158096551895142, |
| "rewards/reward_func/mean": 1.048694372177124, |
| "rewards/reward_func/std": 0.5158095955848694, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.7785451412200928, |
| "sampling/importance_sampling_ratio/mean": 0.9890033602714539, |
| "sampling/importance_sampling_ratio/min": 0.31890588998794556, |
| "sampling/sampling_logp_difference/max": 1.1428592205047607, |
| "sampling/sampling_logp_difference/mean": 0.02159862592816353, |
| "step": 248, |
| "step_time": 215.50420774670783 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14278.0, |
| "completions/max_terminated_length": 14278.0, |
| "completions/mean_length": 9695.75, |
| "completions/mean_terminated_length": 9695.75, |
| "completions/min_length": 4891.0, |
| "completions/min_terminated_length": 4891.0, |
| "entropy": 0.2848109360784292, |
| "epoch": 0.29891956782713086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38538479804992676, |
| "learning_rate": 8e-09, |
| "loss": -0.0234, |
| "num_tokens": 84425578.0, |
| "reward": 1.7004798650741577, |
| "reward_std": 2.567974805831909, |
| "rewards/reward_func/mean": 1.7004798650741577, |
| "rewards/reward_func/std": 2.567974805831909, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.873475432395935, |
| "sampling/importance_sampling_ratio/mean": 0.9899736642837524, |
| "sampling/importance_sampling_ratio/min": 0.29256924986839294, |
| "sampling/sampling_logp_difference/max": 1.2290538549423218, |
| "sampling/sampling_logp_difference/mean": 0.01992029696702957, |
| "step": 249, |
| "step_time": 247.963375205989 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11532.0, |
| "completions/max_terminated_length": 11532.0, |
| "completions/mean_length": 9036.21875, |
| "completions/mean_terminated_length": 9036.21875, |
| "completions/min_length": 4886.0, |
| "completions/min_terminated_length": 4886.0, |
| "entropy": 0.27648542169481516, |
| "epoch": 0.30012004801920766, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8751509189605713, |
| "learning_rate": 4e-09, |
| "loss": -0.0151, |
| "num_tokens": 84746073.0, |
| "reward": 3.147611618041992, |
| "reward_std": 4.971174716949463, |
| "rewards/reward_func/mean": 3.147611618041992, |
| "rewards/reward_func/std": 4.971174716949463, |
| "rewards/soft_overlong_punishment_reward/mean": 0.0, |
| "rewards/soft_overlong_punishment_reward/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.4234373569488525, |
| "sampling/importance_sampling_ratio/mean": 0.9902631640434265, |
| "sampling/importance_sampling_ratio/min": 0.09392388164997101, |
| "sampling/sampling_logp_difference/max": 2.3652706146240234, |
| "sampling/sampling_logp_difference/mean": 0.019402729347348213, |
| "step": 250, |
| "step_time": 203.76658588682767 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 250, |
| "num_input_tokens_seen": 84746073, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
