| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.3333333333333333, |
| "eval_steps": 500, |
| "global_step": 200, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 182.0, |
| "completions/max_terminated_length": 182.0, |
| "completions/mean_length": 137.25, |
| "completions/mean_terminated_length": 137.25, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.2721036672592163, |
| "epoch": 0.0016666666666666668, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-06, |
| "loss": 0.0, |
| "num_tokens": 3013.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 1, |
| "step_time": 26.99397590800072 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 125.0, |
| "completions/max_terminated_length": 125.0, |
| "completions/mean_length": 114.0, |
| "completions/mean_terminated_length": 114.0, |
| "completions/min_length": 98.0, |
| "completions/min_terminated_length": 98.0, |
| "entropy": 0.901360273361206, |
| "epoch": 0.0033333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.376953125, |
| "learning_rate": 4.975000000000001e-06, |
| "loss": -0.04769711568951607, |
| "num_tokens": 5917.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 2, |
| "step_time": 19.053884660999756 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 197.0, |
| "completions/max_terminated_length": 197.0, |
| "completions/mean_length": 143.75, |
| "completions/mean_terminated_length": 143.75, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.1808185577392578, |
| "epoch": 0.005, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.28515625, |
| "learning_rate": 4.95e-06, |
| "loss": -0.011300479993224144, |
| "num_tokens": 8552.0, |
| "reward": 0.4793750047683716, |
| "reward_std": 0.28874996304512024, |
| "rewards/reward_fn/mean": 0.4793750047683716, |
| "rewards/reward_fn/std": 0.288750022649765, |
| "step": 3, |
| "step_time": 28.53829234299701 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 132.0, |
| "completions/max_terminated_length": 132.0, |
| "completions/mean_length": 115.5, |
| "completions/mean_terminated_length": 115.5, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.0730382204055786, |
| "epoch": 0.006666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.32421875, |
| "learning_rate": 4.925e-06, |
| "loss": 0.07140593230724335, |
| "num_tokens": 11250.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 4, |
| "step_time": 20.0260389340001 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 125.5, |
| "completions/mean_terminated_length": 125.5, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.044655680656433, |
| "epoch": 0.008333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.30078125, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.005974027793854475, |
| "num_tokens": 14256.0, |
| "reward": 0.48374998569488525, |
| "reward_std": 0.29750001430511475, |
| "rewards/reward_fn/mean": 0.48374998569488525, |
| "rewards/reward_fn/std": 0.29750001430511475, |
| "step": 5, |
| "step_time": 22.637786576000508 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 175.0, |
| "completions/max_terminated_length": 175.0, |
| "completions/mean_length": 146.5, |
| "completions/mean_terminated_length": 146.5, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.157387137413025, |
| "epoch": 0.01, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.30859375, |
| "learning_rate": 4.875e-06, |
| "loss": -0.018765343353152275, |
| "num_tokens": 17146.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 6, |
| "step_time": 25.646922176998487 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 185.0, |
| "completions/max_terminated_length": 185.0, |
| "completions/mean_length": 167.25, |
| "completions/mean_terminated_length": 167.25, |
| "completions/min_length": 135.0, |
| "completions/min_terminated_length": 135.0, |
| "entropy": 1.1936835050582886, |
| "epoch": 0.011666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.28515625, |
| "learning_rate": 4.85e-06, |
| "loss": 0.04119409993290901, |
| "num_tokens": 20239.0, |
| "reward": 0.6412500143051147, |
| "reward_std": 0.35391560196876526, |
| "rewards/reward_fn/mean": 0.6412500143051147, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 7, |
| "step_time": 27.48416990199985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 221.0, |
| "completions/max_terminated_length": 221.0, |
| "completions/mean_length": 168.0, |
| "completions/mean_terminated_length": 168.0, |
| "completions/min_length": 136.0, |
| "completions/min_terminated_length": 136.0, |
| "entropy": 1.0776662826538086, |
| "epoch": 0.013333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.23046875, |
| "learning_rate": 4.825e-06, |
| "loss": 0.08870118111371994, |
| "num_tokens": 22611.0, |
| "reward": 0.500374972820282, |
| "reward_std": 0.31010571122169495, |
| "rewards/reward_fn/mean": 0.500374972820282, |
| "rewards/reward_fn/std": 0.31010571122169495, |
| "step": 8, |
| "step_time": 31.74235246600074 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 212.0, |
| "completions/max_terminated_length": 212.0, |
| "completions/mean_length": 159.25, |
| "completions/mean_terminated_length": 159.25, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.1662333011627197, |
| "epoch": 0.015, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": 0.0, |
| "num_tokens": 25760.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 9, |
| "step_time": 31.728966909002338 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 121.5, |
| "completions/mean_terminated_length": 121.5, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 1.0164639949798584, |
| "epoch": 0.016666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.306640625, |
| "learning_rate": 4.775e-06, |
| "loss": 0.1400359570980072, |
| "num_tokens": 28494.0, |
| "reward": 0.7856249809265137, |
| "reward_std": 0.3052483797073364, |
| "rewards/reward_fn/mean": 0.7856249809265137, |
| "rewards/reward_fn/std": 0.3052483797073364, |
| "step": 10, |
| "step_time": 23.630102548999275 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 183.0, |
| "completions/max_terminated_length": 183.0, |
| "completions/mean_length": 156.0, |
| "completions/mean_terminated_length": 156.0, |
| "completions/min_length": 127.0, |
| "completions/min_terminated_length": 127.0, |
| "entropy": 1.2048428058624268, |
| "epoch": 0.018333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.306640625, |
| "learning_rate": 4.75e-06, |
| "loss": 0.04951930046081543, |
| "num_tokens": 31562.0, |
| "reward": 0.8075000047683716, |
| "reward_std": 0.2921329736709595, |
| "rewards/reward_fn/mean": 0.8075000047683716, |
| "rewards/reward_fn/std": 0.2921329438686371, |
| "step": 11, |
| "step_time": 28.393185649998486 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 163.0, |
| "completions/max_terminated_length": 163.0, |
| "completions/mean_length": 116.5, |
| "completions/mean_terminated_length": 116.5, |
| "completions/min_length": 82.0, |
| "completions/min_terminated_length": 82.0, |
| "entropy": 1.0292104482650757, |
| "epoch": 0.02, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 4.7250000000000005e-06, |
| "loss": -0.14611932635307312, |
| "num_tokens": 34288.0, |
| "reward": 0.33125001192092896, |
| "reward_std": 0.007500012870877981, |
| "rewards/reward_fn/mean": 0.33125001192092896, |
| "rewards/reward_fn/std": 0.007500007748603821, |
| "step": 12, |
| "step_time": 25.381137749998743 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 157.0, |
| "completions/max_terminated_length": 157.0, |
| "completions/mean_length": 146.0, |
| "completions/mean_terminated_length": 146.0, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.2967694997787476, |
| "epoch": 0.021666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.306640625, |
| "learning_rate": 4.7e-06, |
| "loss": 0.04130696505308151, |
| "num_tokens": 37036.0, |
| "reward": 0.6412500143051147, |
| "reward_std": 0.35391560196876526, |
| "rewards/reward_fn/mean": 0.6412500143051147, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 13, |
| "step_time": 24.01959514600094 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 181.0, |
| "completions/max_terminated_length": 181.0, |
| "completions/mean_length": 163.0, |
| "completions/mean_terminated_length": 163.0, |
| "completions/min_length": 150.0, |
| "completions/min_terminated_length": 150.0, |
| "entropy": 1.2760556936264038, |
| "epoch": 0.023333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.279296875, |
| "learning_rate": 4.675000000000001e-06, |
| "loss": 0.03943007439374924, |
| "num_tokens": 40060.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 14, |
| "step_time": 26.05132091699852 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 163.0, |
| "completions/max_terminated_length": 163.0, |
| "completions/mean_length": 154.0, |
| "completions/mean_terminated_length": 154.0, |
| "completions/min_length": 145.0, |
| "completions/min_terminated_length": 145.0, |
| "entropy": 1.0788471698760986, |
| "epoch": 0.025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.291015625, |
| "learning_rate": 4.65e-06, |
| "loss": -0.024401623755693436, |
| "num_tokens": 42856.0, |
| "reward": 0.9606249928474426, |
| "reward_std": 0.02202034927904606, |
| "rewards/reward_fn/mean": 0.9606249928474426, |
| "rewards/reward_fn/std": 0.02202034927904606, |
| "step": 15, |
| "step_time": 24.084032504997595 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 257.0, |
| "completions/max_terminated_length": 257.0, |
| "completions/mean_length": 193.5, |
| "completions/mean_terminated_length": 193.5, |
| "completions/min_length": 159.0, |
| "completions/min_terminated_length": 159.0, |
| "entropy": 1.4748042821884155, |
| "epoch": 0.02666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.275390625, |
| "learning_rate": 4.625000000000001e-06, |
| "loss": 0.10702173411846161, |
| "num_tokens": 46438.0, |
| "reward": 0.6493749618530273, |
| "reward_std": 0.355049192905426, |
| "rewards/reward_fn/mean": 0.6493749618530273, |
| "rewards/reward_fn/std": 0.35504916310310364, |
| "step": 16, |
| "step_time": 37.7986828969988 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 143.5, |
| "completions/mean_terminated_length": 143.5, |
| "completions/min_length": 117.0, |
| "completions/min_terminated_length": 117.0, |
| "entropy": 1.177292823791504, |
| "epoch": 0.028333333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.330078125, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.04349641874432564, |
| "num_tokens": 48760.0, |
| "reward": 0.9275000095367432, |
| "reward_std": 0.07499998807907104, |
| "rewards/reward_fn/mean": 0.9275000095367432, |
| "rewards/reward_fn/std": 0.07499998807907104, |
| "step": 17, |
| "step_time": 22.96707353400052 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 122.0, |
| "completions/max_terminated_length": 122.0, |
| "completions/mean_length": 108.75, |
| "completions/mean_terminated_length": 108.75, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 0.9841785430908203, |
| "epoch": 0.03, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4375, |
| "learning_rate": 4.575e-06, |
| "loss": -0.0374990850687027, |
| "num_tokens": 51631.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 18, |
| "step_time": 18.896498049998627 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 125.0, |
| "completions/max_terminated_length": 125.0, |
| "completions/mean_length": 116.0, |
| "completions/mean_terminated_length": 116.0, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.1009763479232788, |
| "epoch": 0.03166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.375, |
| "learning_rate": 4.5500000000000005e-06, |
| "loss": -0.02571682445704937, |
| "num_tokens": 54431.0, |
| "reward": 0.9562499523162842, |
| "reward_std": 0.017499983310699463, |
| "rewards/reward_fn/mean": 0.9562499523162842, |
| "rewards/reward_fn/std": 0.017499983310699463, |
| "step": 19, |
| "step_time": 19.452795449000405 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 187.0, |
| "completions/max_terminated_length": 187.0, |
| "completions/mean_length": 140.75, |
| "completions/mean_terminated_length": 140.75, |
| "completions/min_length": 85.0, |
| "completions/min_terminated_length": 85.0, |
| "entropy": 1.152324914932251, |
| "epoch": 0.03333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.361328125, |
| "learning_rate": 4.525000000000001e-06, |
| "loss": 0.1016233041882515, |
| "num_tokens": 57738.0, |
| "reward": 0.953249990940094, |
| "reward_std": 0.01649998500943184, |
| "rewards/reward_fn/mean": 0.953249990940094, |
| "rewards/reward_fn/std": 0.01649998500943184, |
| "step": 20, |
| "step_time": 28.531233167999744 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 144.0, |
| "completions/max_terminated_length": 144.0, |
| "completions/mean_length": 121.0, |
| "completions/mean_terminated_length": 121.0, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 1.0693097114562988, |
| "epoch": 0.035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39453125, |
| "learning_rate": 4.5e-06, |
| "loss": 0.05338107421994209, |
| "num_tokens": 60750.0, |
| "reward": 0.9728749990463257, |
| "reward_std": 0.015750011429190636, |
| "rewards/reward_fn/mean": 0.9728749990463257, |
| "rewards/reward_fn/std": 0.015750020742416382, |
| "step": 21, |
| "step_time": 22.120251809003094 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 146.0, |
| "completions/max_terminated_length": 146.0, |
| "completions/mean_length": 122.0, |
| "completions/mean_terminated_length": 122.0, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.0988726615905762, |
| "epoch": 0.03666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37890625, |
| "learning_rate": 4.475e-06, |
| "loss": -0.032417263835668564, |
| "num_tokens": 63754.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 22, |
| "step_time": 22.679130085998622 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 153.0, |
| "completions/max_terminated_length": 153.0, |
| "completions/mean_length": 121.5, |
| "completions/mean_terminated_length": 121.5, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 1.1439276933670044, |
| "epoch": 0.03833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.306640625, |
| "learning_rate": 4.450000000000001e-06, |
| "loss": -0.07120607793331146, |
| "num_tokens": 66768.0, |
| "reward": 0.9781249761581421, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9781249761581421, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 23, |
| "step_time": 23.13522673600164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 136.0, |
| "completions/max_terminated_length": 136.0, |
| "completions/mean_length": 120.25, |
| "completions/mean_terminated_length": 120.25, |
| "completions/min_length": 102.0, |
| "completions/min_terminated_length": 102.0, |
| "entropy": 1.2446463108062744, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38671875, |
| "learning_rate": 4.425e-06, |
| "loss": 0.04352416470646858, |
| "num_tokens": 69557.0, |
| "reward": 0.7987499833106995, |
| "reward_std": 0.3096066117286682, |
| "rewards/reward_fn/mean": 0.7987499833106995, |
| "rewards/reward_fn/std": 0.3096065819263458, |
| "step": 24, |
| "step_time": 20.66519093700117 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 153.0, |
| "completions/max_terminated_length": 153.0, |
| "completions/mean_length": 139.5, |
| "completions/mean_terminated_length": 139.5, |
| "completions/min_length": 111.0, |
| "completions/min_terminated_length": 111.0, |
| "entropy": 1.232834815979004, |
| "epoch": 0.041666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3671875, |
| "learning_rate": 4.4e-06, |
| "loss": -0.06304893642663956, |
| "num_tokens": 72543.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.3441172242164612, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.3441172242164612, |
| "step": 25, |
| "step_time": 22.765788336000696 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 161.0, |
| "completions/max_terminated_length": 161.0, |
| "completions/mean_length": 138.5, |
| "completions/mean_terminated_length": 138.5, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 1.2499821186065674, |
| "epoch": 0.043333333333333335, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 4.3750000000000005e-06, |
| "loss": 0.0, |
| "num_tokens": 74829.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 26, |
| "step_time": 23.262686510002823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 167.0, |
| "completions/max_terminated_length": 167.0, |
| "completions/mean_length": 134.75, |
| "completions/mean_terminated_length": 134.75, |
| "completions/min_length": 118.0, |
| "completions/min_terminated_length": 118.0, |
| "entropy": 1.2326076030731201, |
| "epoch": 0.045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40234375, |
| "learning_rate": 4.350000000000001e-06, |
| "loss": -0.062136210501194, |
| "num_tokens": 78028.0, |
| "reward": 0.7699999809265137, |
| "reward_std": 0.38999998569488525, |
| "rewards/reward_fn/mean": 0.7699999809265137, |
| "rewards/reward_fn/std": 0.38999998569488525, |
| "step": 27, |
| "step_time": 25.535409896998317 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 106.5, |
| "completions/mean_terminated_length": 106.5, |
| "completions/min_length": 83.0, |
| "completions/min_terminated_length": 83.0, |
| "entropy": 1.0672565698623657, |
| "epoch": 0.04666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.361328125, |
| "learning_rate": 4.325e-06, |
| "loss": 0.10957296937704086, |
| "num_tokens": 80670.0, |
| "reward": 0.9722499847412109, |
| "reward_std": 0.014500022865831852, |
| "rewards/reward_fn/mean": 0.9722499847412109, |
| "rewards/reward_fn/std": 0.014500021934509277, |
| "step": 28, |
| "step_time": 22.534873515000072 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 206.0, |
| "completions/max_terminated_length": 206.0, |
| "completions/mean_length": 138.0, |
| "completions/mean_terminated_length": 138.0, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.063458800315857, |
| "epoch": 0.04833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.298828125, |
| "learning_rate": 4.3e-06, |
| "loss": -0.1097787395119667, |
| "num_tokens": 83466.0, |
| "reward": 0.8206249475479126, |
| "reward_std": 0.3005298972129822, |
| "rewards/reward_fn/mean": 0.8206249475479126, |
| "rewards/reward_fn/std": 0.3005298972129822, |
| "step": 29, |
| "step_time": 31.418785993999336 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 170.0, |
| "completions/max_terminated_length": 170.0, |
| "completions/mean_length": 146.0, |
| "completions/mean_terminated_length": 146.0, |
| "completions/min_length": 127.0, |
| "completions/min_terminated_length": 127.0, |
| "entropy": 1.1856608390808105, |
| "epoch": 0.05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3046875, |
| "learning_rate": 4.2750000000000006e-06, |
| "loss": 0.020558878779411316, |
| "num_tokens": 86306.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.010103637352585793, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.010103637352585793, |
| "step": 30, |
| "step_time": 25.58633564399861 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 179.0, |
| "completions/max_terminated_length": 179.0, |
| "completions/mean_length": 134.0, |
| "completions/mean_terminated_length": 134.0, |
| "completions/min_length": 86.0, |
| "completions/min_terminated_length": 86.0, |
| "entropy": 1.0142335891723633, |
| "epoch": 0.051666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.28515625, |
| "learning_rate": 4.25e-06, |
| "loss": -0.1461215764284134, |
| "num_tokens": 89374.0, |
| "reward": 0.33937501907348633, |
| "reward_std": 0.022020353004336357, |
| "rewards/reward_fn/mean": 0.33937501907348633, |
| "rewards/reward_fn/std": 0.022020353004336357, |
| "step": 31, |
| "step_time": 26.99719125000047 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 170.0, |
| "completions/max_terminated_length": 170.0, |
| "completions/mean_length": 139.0, |
| "completions/mean_terminated_length": 139.0, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "entropy": 1.3681882619857788, |
| "epoch": 0.05333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2734375, |
| "learning_rate": 4.225e-06, |
| "loss": -0.09904350340366364, |
| "num_tokens": 92234.0, |
| "reward": 0.6587499976158142, |
| "reward_std": 0.35391557216644287, |
| "rewards/reward_fn/mean": 0.6587499976158142, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 32, |
| "step_time": 25.45712414800073 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 169.0, |
| "completions/max_terminated_length": 169.0, |
| "completions/mean_length": 110.75, |
| "completions/mean_terminated_length": 110.75, |
| "completions/min_length": 65.0, |
| "completions/min_terminated_length": 65.0, |
| "entropy": 1.112715721130371, |
| "epoch": 0.055, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": -0.13209989666938782, |
| "num_tokens": 94781.0, |
| "reward": 0.37325000762939453, |
| "reward_std": 0.04913502559065819, |
| "rewards/reward_fn/mean": 0.37325000762939453, |
| "rewards/reward_fn/std": 0.04913502186536789, |
| "step": 33, |
| "step_time": 24.924523759000294 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 196.0, |
| "completions/max_terminated_length": 196.0, |
| "completions/mean_length": 169.0, |
| "completions/mean_terminated_length": 169.0, |
| "completions/min_length": 149.0, |
| "completions/min_terminated_length": 149.0, |
| "entropy": 1.3067529201507568, |
| "epoch": 0.056666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.298828125, |
| "learning_rate": 4.175e-06, |
| "loss": -0.0015746492426842451, |
| "num_tokens": 97417.0, |
| "reward": 0.5913749933242798, |
| "reward_std": 0.31556180119514465, |
| "rewards/reward_fn/mean": 0.5913749933242798, |
| "rewards/reward_fn/std": 0.31556177139282227, |
| "step": 34, |
| "step_time": 29.42368617000102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 128.0, |
| "completions/mean_terminated_length": 128.0, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 1.064958930015564, |
| "epoch": 0.058333333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 4.15e-06, |
| "loss": 0.0, |
| "num_tokens": 100201.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 35, |
| "step_time": 22.600620116001664 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 133.0, |
| "completions/max_terminated_length": 133.0, |
| "completions/mean_length": 115.25, |
| "completions/mean_terminated_length": 115.25, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 1.1022453308105469, |
| "epoch": 0.06, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.357421875, |
| "learning_rate": 4.125e-06, |
| "loss": -0.03460807725787163, |
| "num_tokens": 102946.0, |
| "reward": 0.6850000023841858, |
| "reward_std": 0.32457661628723145, |
| "rewards/reward_fn/mean": 0.6850000023841858, |
| "rewards/reward_fn/std": 0.32457664608955383, |
| "step": 36, |
| "step_time": 19.86391903600088 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 181.0, |
| "completions/max_terminated_length": 181.0, |
| "completions/mean_length": 152.5, |
| "completions/mean_terminated_length": 152.5, |
| "completions/min_length": 125.0, |
| "completions/min_terminated_length": 125.0, |
| "entropy": 1.1503243446350098, |
| "epoch": 0.06166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29296875, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0848444476723671, |
| "num_tokens": 105380.0, |
| "reward": 0.6150000095367432, |
| "reward_std": 0.34411725401878357, |
| "rewards/reward_fn/mean": 0.6150000095367432, |
| "rewards/reward_fn/std": 0.3441172242164612, |
| "step": 37, |
| "step_time": 25.59769222900286 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 116.0, |
| "completions/max_terminated_length": 116.0, |
| "completions/mean_length": 113.5, |
| "completions/mean_terminated_length": 113.5, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.0994595289230347, |
| "epoch": 0.06333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.400390625, |
| "learning_rate": 4.075e-06, |
| "loss": 0.016566012054681778, |
| "num_tokens": 108438.0, |
| "reward": 0.9772499799728394, |
| "reward_std": 0.015256185084581375, |
| "rewards/reward_fn/mean": 0.9772499799728394, |
| "rewards/reward_fn/std": 0.015256169252097607, |
| "step": 38, |
| "step_time": 17.853799561002234 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 187.0, |
| "completions/max_terminated_length": 187.0, |
| "completions/mean_length": 139.25, |
| "completions/mean_terminated_length": 139.25, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.2051821947097778, |
| "epoch": 0.065, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.302734375, |
| "learning_rate": 4.05e-06, |
| "loss": -0.16876700520515442, |
| "num_tokens": 111567.0, |
| "reward": 0.48375001549720764, |
| "reward_std": 0.3209393620491028, |
| "rewards/reward_fn/mean": 0.48375001549720764, |
| "rewards/reward_fn/std": 0.3209393620491028, |
| "step": 39, |
| "step_time": 27.307055137000134 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 174.0, |
| "completions/max_terminated_length": 174.0, |
| "completions/mean_length": 132.5, |
| "completions/mean_terminated_length": 132.5, |
| "completions/min_length": 105.0, |
| "completions/min_terminated_length": 105.0, |
| "entropy": 1.0574369430541992, |
| "epoch": 0.06666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.328125, |
| "learning_rate": 4.0250000000000004e-06, |
| "loss": 0.09384524822235107, |
| "num_tokens": 114485.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.010103637352585793, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.010103637352585793, |
| "step": 40, |
| "step_time": 25.46330049500102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 138.0, |
| "completions/max_terminated_length": 138.0, |
| "completions/mean_length": 127.5, |
| "completions/mean_terminated_length": 127.5, |
| "completions/min_length": 119.0, |
| "completions/min_terminated_length": 119.0, |
| "entropy": 1.045518159866333, |
| "epoch": 0.06833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3359375, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.001768031739629805, |
| "num_tokens": 117383.0, |
| "reward": 0.29750001430511475, |
| "reward_std": 0.09970790892839432, |
| "rewards/reward_fn/mean": 0.29750001430511475, |
| "rewards/reward_fn/std": 0.09970790892839432, |
| "step": 41, |
| "step_time": 20.58559491000051 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 136.75, |
| "completions/mean_terminated_length": 136.75, |
| "completions/min_length": 110.0, |
| "completions/min_terminated_length": 110.0, |
| "entropy": 1.2528449296951294, |
| "epoch": 0.07, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.375, |
| "learning_rate": 3.975000000000001e-06, |
| "loss": -0.0037579911295324564, |
| "num_tokens": 120630.0, |
| "reward": 0.33937501907348633, |
| "reward_std": 0.036077164113521576, |
| "rewards/reward_fn/mean": 0.33937501907348633, |
| "rewards/reward_fn/std": 0.036077164113521576, |
| "step": 42, |
| "step_time": 23.60465663499781 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 149.0, |
| "completions/max_terminated_length": 149.0, |
| "completions/mean_length": 123.5, |
| "completions/mean_terminated_length": 123.5, |
| "completions/min_length": 96.0, |
| "completions/min_terminated_length": 96.0, |
| "entropy": 1.0263532400131226, |
| "epoch": 0.07166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.396484375, |
| "learning_rate": 3.95e-06, |
| "loss": -0.010047557763755322, |
| "num_tokens": 123572.0, |
| "reward": 0.972000002861023, |
| "reward_std": 0.013999998569488525, |
| "rewards/reward_fn/mean": 0.972000002861023, |
| "rewards/reward_fn/std": 0.013999998569488525, |
| "step": 43, |
| "step_time": 21.8416585140003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 179.0, |
| "completions/max_terminated_length": 179.0, |
| "completions/mean_length": 158.25, |
| "completions/mean_terminated_length": 158.25, |
| "completions/min_length": 139.0, |
| "completions/min_terminated_length": 139.0, |
| "entropy": 1.4695942401885986, |
| "epoch": 0.07333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.294921875, |
| "learning_rate": 3.9250000000000005e-06, |
| "loss": -0.07597894221544266, |
| "num_tokens": 126769.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.3441172242164612, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.3441172242164612, |
| "step": 44, |
| "step_time": 26.284113427002012 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 176.0, |
| "completions/max_terminated_length": 176.0, |
| "completions/mean_length": 132.75, |
| "completions/mean_terminated_length": 132.75, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.3445155620574951, |
| "epoch": 0.075, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37890625, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.04559139907360077, |
| "num_tokens": 129512.0, |
| "reward": 0.6675000190734863, |
| "reward_std": 0.34470999240875244, |
| "rewards/reward_fn/mean": 0.6675000190734863, |
| "rewards/reward_fn/std": 0.34470999240875244, |
| "step": 45, |
| "step_time": 26.141577066002355 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 131.0, |
| "completions/max_terminated_length": 131.0, |
| "completions/mean_length": 125.25, |
| "completions/mean_terminated_length": 125.25, |
| "completions/min_length": 115.0, |
| "completions/min_terminated_length": 115.0, |
| "entropy": 1.098323941230774, |
| "epoch": 0.07666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.349609375, |
| "learning_rate": 3.875e-06, |
| "loss": -0.021484289318323135, |
| "num_tokens": 132537.0, |
| "reward": 0.49162501096725464, |
| "reward_std": 0.3345796763896942, |
| "rewards/reward_fn/mean": 0.49162501096725464, |
| "rewards/reward_fn/std": 0.3345796763896942, |
| "step": 46, |
| "step_time": 20.43934466399878 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 127.75, |
| "completions/mean_terminated_length": 127.75, |
| "completions/min_length": 116.0, |
| "completions/min_terminated_length": 116.0, |
| "entropy": 1.1513502597808838, |
| "epoch": 0.07833333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3.85e-06, |
| "loss": 0.0, |
| "num_tokens": 135108.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 47, |
| "step_time": 24.161369523997564 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 139.0, |
| "completions/mean_terminated_length": 139.0, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.0865966081619263, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.296875, |
| "learning_rate": 3.825000000000001e-06, |
| "loss": -0.03219185769557953, |
| "num_tokens": 137804.0, |
| "reward": 0.9562499523162842, |
| "reward_std": 0.017499983310699463, |
| "rewards/reward_fn/mean": 0.9562499523162842, |
| "rewards/reward_fn/std": 0.017499983310699463, |
| "step": 48, |
| "step_time": 23.38693464699827 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 144.0, |
| "completions/max_terminated_length": 144.0, |
| "completions/mean_length": 116.75, |
| "completions/mean_terminated_length": 116.75, |
| "completions/min_length": 104.0, |
| "completions/min_terminated_length": 104.0, |
| "entropy": 1.214436650276184, |
| "epoch": 0.08166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.35546875, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": 0.026122601702809334, |
| "num_tokens": 140483.0, |
| "reward": 0.9518749713897705, |
| "reward_std": 0.01675495319068432, |
| "rewards/reward_fn/mean": 0.9518749713897705, |
| "rewards/reward_fn/std": 0.016754958778619766, |
| "step": 49, |
| "step_time": 22.844522692001192 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 162.0, |
| "completions/max_terminated_length": 162.0, |
| "completions/mean_length": 135.0, |
| "completions/mean_terminated_length": 135.0, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.2262229919433594, |
| "epoch": 0.08333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.333984375, |
| "learning_rate": 3.7750000000000003e-06, |
| "loss": -0.003206648863852024, |
| "num_tokens": 143371.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.36373066902160645, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.36373066902160645, |
| "step": 50, |
| "step_time": 24.662258178999764 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 165.0, |
| "completions/max_terminated_length": 165.0, |
| "completions/mean_length": 127.5, |
| "completions/mean_terminated_length": 127.5, |
| "completions/min_length": 102.0, |
| "completions/min_terminated_length": 102.0, |
| "entropy": 1.0940418243408203, |
| "epoch": 0.085, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 3.7500000000000005e-06, |
| "loss": -0.10688165575265884, |
| "num_tokens": 146097.0, |
| "reward": 0.9781249761581421, |
| "reward_std": 0.016754984855651855, |
| "rewards/reward_fn/mean": 0.9781249761581421, |
| "rewards/reward_fn/std": 0.016754990443587303, |
| "step": 51, |
| "step_time": 25.447743856999296 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 178.0, |
| "completions/max_terminated_length": 178.0, |
| "completions/mean_length": 148.0, |
| "completions/mean_terminated_length": 148.0, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.1700999736785889, |
| "epoch": 0.08666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29296875, |
| "learning_rate": 3.7250000000000003e-06, |
| "loss": -0.0370216965675354, |
| "num_tokens": 149049.0, |
| "reward": 0.9518749713897705, |
| "reward_std": 0.026249965652823448, |
| "rewards/reward_fn/mean": 0.9518749713897705, |
| "rewards/reward_fn/std": 0.026249976828694344, |
| "step": 52, |
| "step_time": 26.492212680001103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 179.0, |
| "completions/max_terminated_length": 179.0, |
| "completions/mean_length": 145.75, |
| "completions/mean_terminated_length": 145.75, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "entropy": 1.0578383207321167, |
| "epoch": 0.08833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.322265625, |
| "learning_rate": 3.7e-06, |
| "loss": -0.04324590787291527, |
| "num_tokens": 151888.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 53, |
| "step_time": 26.41010283899959 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 147.0, |
| "completions/max_terminated_length": 147.0, |
| "completions/mean_length": 131.75, |
| "completions/mean_terminated_length": 131.75, |
| "completions/min_length": 117.0, |
| "completions/min_terminated_length": 117.0, |
| "entropy": 1.2011436223983765, |
| "epoch": 0.09, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3125, |
| "learning_rate": 3.6750000000000004e-06, |
| "loss": 0.05730345472693443, |
| "num_tokens": 154483.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.3441172242164612, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.3441172242164612, |
| "step": 54, |
| "step_time": 21.763525555998058 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 136.0, |
| "completions/max_terminated_length": 136.0, |
| "completions/mean_length": 127.75, |
| "completions/mean_terminated_length": 127.75, |
| "completions/min_length": 115.0, |
| "completions/min_terminated_length": 115.0, |
| "entropy": 1.1653622388839722, |
| "epoch": 0.09166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.31640625, |
| "learning_rate": 3.65e-06, |
| "loss": -0.031099891290068626, |
| "num_tokens": 157822.0, |
| "reward": 0.6062500476837158, |
| "reward_std": 0.29886937141418457, |
| "rewards/reward_fn/mean": 0.6062500476837158, |
| "rewards/reward_fn/std": 0.29886940121650696, |
| "step": 55, |
| "step_time": 21.180196135002916 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 125.0, |
| "completions/max_terminated_length": 125.0, |
| "completions/mean_length": 110.75, |
| "completions/mean_terminated_length": 110.75, |
| "completions/min_length": 94.0, |
| "completions/min_terminated_length": 94.0, |
| "entropy": 0.9973193407058716, |
| "epoch": 0.09333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.400390625, |
| "learning_rate": 3.625e-06, |
| "loss": 0.044428952038288116, |
| "num_tokens": 160477.0, |
| "reward": 0.3193749785423279, |
| "reward_std": 0.01641835831105709, |
| "rewards/reward_fn/mean": 0.3193749785423279, |
| "rewards/reward_fn/std": 0.01641835644841194, |
| "step": 56, |
| "step_time": 19.055930811002327 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 137.0, |
| "completions/max_terminated_length": 137.0, |
| "completions/mean_length": 117.5, |
| "completions/mean_terminated_length": 117.5, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.0432900190353394, |
| "epoch": 0.095, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": 0.0, |
| "num_tokens": 163015.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 57, |
| "step_time": 20.755967883000267 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 149.0, |
| "completions/max_terminated_length": 149.0, |
| "completions/mean_length": 127.5, |
| "completions/mean_terminated_length": 127.5, |
| "completions/min_length": 108.0, |
| "completions/min_terminated_length": 108.0, |
| "entropy": 1.0790032148361206, |
| "epoch": 0.09666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.392578125, |
| "learning_rate": 3.575e-06, |
| "loss": 0.08609388768672943, |
| "num_tokens": 165757.0, |
| "reward": 0.33937501907348633, |
| "reward_std": 0.022020353004336357, |
| "rewards/reward_fn/mean": 0.33937501907348633, |
| "rewards/reward_fn/std": 0.022020353004336357, |
| "step": 58, |
| "step_time": 22.144303656001284 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 201.0, |
| "completions/max_terminated_length": 201.0, |
| "completions/mean_length": 155.5, |
| "completions/mean_terminated_length": 155.5, |
| "completions/min_length": 119.0, |
| "completions/min_terminated_length": 119.0, |
| "entropy": 1.3297035694122314, |
| "epoch": 0.09833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 3.5500000000000003e-06, |
| "loss": 0.07765334099531174, |
| "num_tokens": 168967.0, |
| "reward": 0.7524999976158142, |
| "reward_std": 0.2626309096813202, |
| "rewards/reward_fn/mean": 0.7524999976158142, |
| "rewards/reward_fn/std": 0.2626309096813202, |
| "step": 59, |
| "step_time": 29.628764026998397 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 262.0, |
| "completions/max_terminated_length": 262.0, |
| "completions/mean_length": 181.75, |
| "completions/mean_terminated_length": 181.75, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.1880607604980469, |
| "epoch": 0.1, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.24609375, |
| "learning_rate": 3.525e-06, |
| "loss": -0.07361559569835663, |
| "num_tokens": 172254.0, |
| "reward": 0.953374981880188, |
| "reward_std": 0.01649935357272625, |
| "rewards/reward_fn/mean": 0.953374981880188, |
| "rewards/reward_fn/std": 0.01649935357272625, |
| "step": 60, |
| "step_time": 37.99089504899894 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 193.0, |
| "completions/max_terminated_length": 193.0, |
| "completions/mean_length": 125.5, |
| "completions/mean_terminated_length": 125.5, |
| "completions/min_length": 96.0, |
| "completions/min_terminated_length": 96.0, |
| "entropy": 1.1564736366271973, |
| "epoch": 0.10166666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0, |
| "num_tokens": 175288.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 61, |
| "step_time": 28.490477613999246 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 135.0, |
| "completions/max_terminated_length": 135.0, |
| "completions/mean_length": 124.0, |
| "completions/mean_terminated_length": 124.0, |
| "completions/min_length": 117.0, |
| "completions/min_terminated_length": 117.0, |
| "entropy": 1.0834579467773438, |
| "epoch": 0.10333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3359375, |
| "learning_rate": 3.475e-06, |
| "loss": -0.01383125875145197, |
| "num_tokens": 177812.0, |
| "reward": 0.9737499952316284, |
| "reward_std": 0.010103654116392136, |
| "rewards/reward_fn/mean": 0.9737499952316284, |
| "rewards/reward_fn/std": 0.010103654116392136, |
| "step": 62, |
| "step_time": 20.202116815002228 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 186.0, |
| "completions/max_terminated_length": 186.0, |
| "completions/mean_length": 156.75, |
| "completions/mean_terminated_length": 156.75, |
| "completions/min_length": 132.0, |
| "completions/min_terminated_length": 132.0, |
| "entropy": 1.3051375150680542, |
| "epoch": 0.105, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.296875, |
| "learning_rate": 3.45e-06, |
| "loss": -0.058121208101511, |
| "num_tokens": 181135.0, |
| "reward": 0.643875002861023, |
| "reward_std": 0.3709380626678467, |
| "rewards/reward_fn/mean": 0.643875002861023, |
| "rewards/reward_fn/std": 0.37093809247016907, |
| "step": 63, |
| "step_time": 27.585933769001713 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 130.0, |
| "completions/max_terminated_length": 130.0, |
| "completions/mean_length": 118.75, |
| "completions/mean_terminated_length": 118.75, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 0.9585571885108948, |
| "epoch": 0.10666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 3.4250000000000007e-06, |
| "loss": -0.024057308211922646, |
| "num_tokens": 184026.0, |
| "reward": 0.32712501287460327, |
| "reward_std": 0.015750011429190636, |
| "rewards/reward_fn/mean": 0.32712501287460327, |
| "rewards/reward_fn/std": 0.015750005841255188, |
| "step": 64, |
| "step_time": 19.43507832999967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 147.0, |
| "completions/max_terminated_length": 147.0, |
| "completions/mean_length": 124.5, |
| "completions/mean_terminated_length": 124.5, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 1.098405122756958, |
| "epoch": 0.10833333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0, |
| "num_tokens": 186604.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 65, |
| "step_time": 21.2074764780009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 200.0, |
| "completions/max_terminated_length": 200.0, |
| "completions/mean_length": 152.25, |
| "completions/mean_terminated_length": 152.25, |
| "completions/min_length": 124.0, |
| "completions/min_terminated_length": 124.0, |
| "entropy": 1.2371915578842163, |
| "epoch": 0.11, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2890625, |
| "learning_rate": 3.3750000000000003e-06, |
| "loss": -0.09480708837509155, |
| "num_tokens": 189773.0, |
| "reward": 0.3525000214576721, |
| "reward_std": 0.020207257941365242, |
| "rewards/reward_fn/mean": 0.3525000214576721, |
| "rewards/reward_fn/std": 0.020207257941365242, |
| "step": 66, |
| "step_time": 29.151788433002366 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 186.0, |
| "completions/max_terminated_length": 186.0, |
| "completions/mean_length": 135.75, |
| "completions/mean_terminated_length": 135.75, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 1.0945377349853516, |
| "epoch": 0.11166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.341796875, |
| "learning_rate": 3.3500000000000005e-06, |
| "loss": 0.18502062559127808, |
| "num_tokens": 192564.0, |
| "reward": 0.8162499666213989, |
| "reward_std": 0.29749998450279236, |
| "rewards/reward_fn/mean": 0.8162499666213989, |
| "rewards/reward_fn/std": 0.29749998450279236, |
| "step": 67, |
| "step_time": 27.474493719000748 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 160.0, |
| "completions/max_terminated_length": 160.0, |
| "completions/mean_length": 139.5, |
| "completions/mean_terminated_length": 139.5, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.2198596000671387, |
| "epoch": 0.11333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.318359375, |
| "learning_rate": 3.3250000000000004e-06, |
| "loss": -0.08495711535215378, |
| "num_tokens": 195718.0, |
| "reward": 0.6412500143051147, |
| "reward_std": 0.35391560196876526, |
| "rewards/reward_fn/mean": 0.6412500143051147, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 68, |
| "step_time": 23.923134037002455 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 167.0, |
| "completions/max_terminated_length": 167.0, |
| "completions/mean_length": 149.5, |
| "completions/mean_terminated_length": 149.5, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.370664358139038, |
| "epoch": 0.115, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.28515625, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": -0.06370344012975693, |
| "num_tokens": 198864.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.36373066902160645, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.36373066902160645, |
| "step": 69, |
| "step_time": 24.89920946699931 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 254.0, |
| "completions/max_terminated_length": 254.0, |
| "completions/mean_length": 174.75, |
| "completions/mean_terminated_length": 174.75, |
| "completions/min_length": 136.0, |
| "completions/min_terminated_length": 136.0, |
| "entropy": 1.2808506488800049, |
| "epoch": 0.11666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.302734375, |
| "learning_rate": 3.2750000000000004e-06, |
| "loss": 0.02983768843114376, |
| "num_tokens": 201847.0, |
| "reward": 0.48899999260902405, |
| "reward_std": 0.3174019455909729, |
| "rewards/reward_fn/mean": 0.48899999260902405, |
| "rewards/reward_fn/std": 0.3174019455909729, |
| "step": 70, |
| "step_time": 37.47509274999902 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 123.0, |
| "completions/max_terminated_length": 123.0, |
| "completions/mean_length": 110.25, |
| "completions/mean_terminated_length": 110.25, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 1.0120017528533936, |
| "epoch": 0.11833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.380859375, |
| "learning_rate": 3.2500000000000002e-06, |
| "loss": -0.03286806866526604, |
| "num_tokens": 204520.0, |
| "reward": 0.824999988079071, |
| "reward_std": 0.2799999713897705, |
| "rewards/reward_fn/mean": 0.824999988079071, |
| "rewards/reward_fn/std": 0.2800000011920929, |
| "step": 71, |
| "step_time": 19.081610291999823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 177.0, |
| "completions/max_terminated_length": 177.0, |
| "completions/mean_length": 156.5, |
| "completions/mean_terminated_length": 156.5, |
| "completions/min_length": 117.0, |
| "completions/min_terminated_length": 117.0, |
| "entropy": 1.1953434944152832, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.302734375, |
| "learning_rate": 3.2250000000000005e-06, |
| "loss": -0.12470149993896484, |
| "num_tokens": 207550.0, |
| "reward": 0.794374942779541, |
| "reward_std": 0.3183445334434509, |
| "rewards/reward_fn/mean": 0.794374942779541, |
| "rewards/reward_fn/std": 0.31834450364112854, |
| "step": 72, |
| "step_time": 26.463810916997318 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 189.0, |
| "completions/max_terminated_length": 189.0, |
| "completions/mean_length": 143.75, |
| "completions/mean_terminated_length": 143.75, |
| "completions/min_length": 108.0, |
| "completions/min_terminated_length": 108.0, |
| "entropy": 1.2760404348373413, |
| "epoch": 0.12166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3359375, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": -0.20025718212127686, |
| "num_tokens": 210661.0, |
| "reward": 0.6324999928474426, |
| "reward_std": 0.34352341294288635, |
| "rewards/reward_fn/mean": 0.6324999928474426, |
| "rewards/reward_fn/std": 0.34352341294288635, |
| "step": 73, |
| "step_time": 28.05823507000241 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 217.0, |
| "completions/max_terminated_length": 217.0, |
| "completions/mean_length": 169.25, |
| "completions/mean_terminated_length": 169.25, |
| "completions/min_length": 135.0, |
| "completions/min_terminated_length": 135.0, |
| "entropy": 1.1246460676193237, |
| "epoch": 0.12333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 3.175e-06, |
| "loss": -0.06791014224290848, |
| "num_tokens": 213910.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 74, |
| "step_time": 31.710728935002408 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 145.0, |
| "completions/max_terminated_length": 145.0, |
| "completions/mean_length": 122.5, |
| "completions/mean_terminated_length": 122.5, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 1.098929762840271, |
| "epoch": 0.125, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3.1500000000000003e-06, |
| "loss": 0.0, |
| "num_tokens": 216608.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 75, |
| "step_time": 21.737217218000296 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 170.0, |
| "completions/max_terminated_length": 170.0, |
| "completions/mean_length": 146.5, |
| "completions/mean_terminated_length": 146.5, |
| "completions/min_length": 104.0, |
| "completions/min_terminated_length": 104.0, |
| "entropy": 1.0769822597503662, |
| "epoch": 0.12666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.279296875, |
| "learning_rate": 3.125e-06, |
| "loss": 0.14531265199184418, |
| "num_tokens": 219434.0, |
| "reward": 0.5012500286102295, |
| "reward_std": 0.3096066117286682, |
| "rewards/reward_fn/mean": 0.5012500286102295, |
| "rewards/reward_fn/std": 0.3096066117286682, |
| "step": 76, |
| "step_time": 25.090116903000308 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 131.0, |
| "completions/max_terminated_length": 131.0, |
| "completions/mean_length": 121.5, |
| "completions/mean_terminated_length": 121.5, |
| "completions/min_length": 113.0, |
| "completions/min_terminated_length": 113.0, |
| "entropy": 0.9762334823608398, |
| "epoch": 0.12833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.38671875, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": -0.00801115296781063, |
| "num_tokens": 222356.0, |
| "reward": 0.6657500267028809, |
| "reward_std": 0.3621724545955658, |
| "rewards/reward_fn/mean": 0.6657500267028809, |
| "rewards/reward_fn/std": 0.3621724843978882, |
| "step": 77, |
| "step_time": 20.045034541999485 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 165.0, |
| "completions/max_terminated_length": 165.0, |
| "completions/mean_length": 124.5, |
| "completions/mean_terminated_length": 124.5, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 1.2096550464630127, |
| "epoch": 0.13, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.296875, |
| "learning_rate": 3.075e-06, |
| "loss": -0.055578697472810745, |
| "num_tokens": 225550.0, |
| "reward": 0.7949999570846558, |
| "reward_std": 0.30700162053108215, |
| "rewards/reward_fn/mean": 0.7949999570846558, |
| "rewards/reward_fn/std": 0.30700162053108215, |
| "step": 78, |
| "step_time": 25.065289067002595 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 185.0, |
| "completions/max_terminated_length": 185.0, |
| "completions/mean_length": 132.25, |
| "completions/mean_terminated_length": 132.25, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 1.2732487916946411, |
| "epoch": 0.13166666666666665, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 3.05e-06, |
| "loss": -0.197392076253891, |
| "num_tokens": 228395.0, |
| "reward": 0.48374998569488525, |
| "reward_std": 0.3212572932243347, |
| "rewards/reward_fn/mean": 0.48374998569488525, |
| "rewards/reward_fn/std": 0.3212572932243347, |
| "step": 79, |
| "step_time": 27.037959209999826 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 136.0, |
| "completions/max_terminated_length": 136.0, |
| "completions/mean_length": 110.5, |
| "completions/mean_terminated_length": 110.5, |
| "completions/min_length": 76.0, |
| "completions/min_terminated_length": 76.0, |
| "entropy": 1.0314747095108032, |
| "epoch": 0.13333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34765625, |
| "learning_rate": 3.0250000000000003e-06, |
| "loss": 0.15605716407299042, |
| "num_tokens": 230893.0, |
| "reward": 0.4865000247955322, |
| "reward_std": 0.30300000309944153, |
| "rewards/reward_fn/mean": 0.4865000247955322, |
| "rewards/reward_fn/std": 0.30300000309944153, |
| "step": 80, |
| "step_time": 20.13105025200275 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 191.0, |
| "completions/max_terminated_length": 191.0, |
| "completions/mean_length": 141.5, |
| "completions/mean_terminated_length": 141.5, |
| "completions/min_length": 108.0, |
| "completions/min_terminated_length": 108.0, |
| "entropy": 1.3124823570251465, |
| "epoch": 0.135, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 3e-06, |
| "loss": 0.0, |
| "num_tokens": 233867.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 81, |
| "step_time": 28.160457204001432 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 167.0, |
| "completions/max_terminated_length": 167.0, |
| "completions/mean_length": 146.0, |
| "completions/mean_terminated_length": 146.0, |
| "completions/min_length": 126.0, |
| "completions/min_terminated_length": 126.0, |
| "entropy": 1.0319082736968994, |
| "epoch": 0.13666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 2.9750000000000003e-06, |
| "loss": 0.02396450564265251, |
| "num_tokens": 236651.0, |
| "reward": 0.8162499666213989, |
| "reward_std": 0.29749998450279236, |
| "rewards/reward_fn/mean": 0.8162499666213989, |
| "rewards/reward_fn/std": 0.29749998450279236, |
| "step": 82, |
| "step_time": 24.461930845001916 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 179.0, |
| "completions/max_terminated_length": 179.0, |
| "completions/mean_length": 149.0, |
| "completions/mean_terminated_length": 149.0, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "entropy": 1.2378220558166504, |
| "epoch": 0.13833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.330078125, |
| "learning_rate": 2.95e-06, |
| "loss": 0.06299065053462982, |
| "num_tokens": 239411.0, |
| "reward": 0.5618749856948853, |
| "reward_std": 0.47596120834350586, |
| "rewards/reward_fn/mean": 0.5618749856948853, |
| "rewards/reward_fn/std": 0.47596120834350586, |
| "step": 83, |
| "step_time": 25.732968376996723 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 191.0, |
| "completions/max_terminated_length": 191.0, |
| "completions/mean_length": 172.75, |
| "completions/mean_terminated_length": 172.75, |
| "completions/min_length": 142.0, |
| "completions/min_terminated_length": 142.0, |
| "entropy": 1.1778147220611572, |
| "epoch": 0.14, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2470703125, |
| "learning_rate": 2.925e-06, |
| "loss": -0.08758159726858139, |
| "num_tokens": 242474.0, |
| "reward": 0.794374942779541, |
| "reward_std": 0.3183445334434509, |
| "rewards/reward_fn/mean": 0.794374942779541, |
| "rewards/reward_fn/std": 0.31834450364112854, |
| "step": 84, |
| "step_time": 28.395576832001098 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 127.5, |
| "completions/mean_terminated_length": 127.5, |
| "completions/min_length": 88.0, |
| "completions/min_terminated_length": 88.0, |
| "entropy": 1.192873477935791, |
| "epoch": 0.14166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.294921875, |
| "learning_rate": 2.9e-06, |
| "loss": -0.15402138233184814, |
| "num_tokens": 245432.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.017500003799796104, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.017499998211860657, |
| "step": 85, |
| "step_time": 23.462746484998206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 195.0, |
| "completions/max_terminated_length": 195.0, |
| "completions/mean_length": 135.75, |
| "completions/mean_terminated_length": 135.75, |
| "completions/min_length": 103.0, |
| "completions/min_terminated_length": 103.0, |
| "entropy": 1.3281114101409912, |
| "epoch": 0.14333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.326171875, |
| "learning_rate": 2.875e-06, |
| "loss": 0.21816286444664001, |
| "num_tokens": 248295.0, |
| "reward": 0.8075000047683716, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.8075000047683716, |
| "rewards/reward_fn/std": 0.3149999976158142, |
| "step": 86, |
| "step_time": 29.33027525199941 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 153.0, |
| "completions/max_terminated_length": 153.0, |
| "completions/mean_length": 124.75, |
| "completions/mean_terminated_length": 124.75, |
| "completions/min_length": 111.0, |
| "completions/min_terminated_length": 111.0, |
| "entropy": 1.168349266052246, |
| "epoch": 0.145, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.30078125, |
| "learning_rate": 2.85e-06, |
| "loss": -0.050082288682460785, |
| "num_tokens": 251030.0, |
| "reward": 0.9474999904632568, |
| "reward_std": 0.0202072411775589, |
| "rewards/reward_fn/mean": 0.9474999904632568, |
| "rewards/reward_fn/std": 0.0202072411775589, |
| "step": 87, |
| "step_time": 22.54528503599795 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 128.0, |
| "completions/max_terminated_length": 128.0, |
| "completions/mean_length": 109.75, |
| "completions/mean_terminated_length": 109.75, |
| "completions/min_length": 90.0, |
| "completions/min_terminated_length": 90.0, |
| "entropy": 0.9876170754432678, |
| "epoch": 0.14666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.345703125, |
| "learning_rate": 2.825e-06, |
| "loss": -0.04842241480946541, |
| "num_tokens": 253533.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 88, |
| "step_time": 18.978010971000913 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 166.0, |
| "completions/max_terminated_length": 166.0, |
| "completions/mean_length": 149.75, |
| "completions/mean_terminated_length": 149.75, |
| "completions/min_length": 138.0, |
| "completions/min_terminated_length": 138.0, |
| "entropy": 1.289428949356079, |
| "epoch": 0.14833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": 0.019071724265813828, |
| "num_tokens": 256532.0, |
| "reward": 0.8206250071525574, |
| "reward_std": 0.3005298972129822, |
| "rewards/reward_fn/mean": 0.8206250071525574, |
| "rewards/reward_fn/std": 0.3005298972129822, |
| "step": 89, |
| "step_time": 25.60237361599866 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 142.0, |
| "completions/max_terminated_length": 142.0, |
| "completions/mean_length": 121.75, |
| "completions/mean_terminated_length": 121.75, |
| "completions/min_length": 103.0, |
| "completions/min_terminated_length": 103.0, |
| "entropy": 1.0802006721496582, |
| "epoch": 0.15, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.361328125, |
| "learning_rate": 2.7750000000000005e-06, |
| "loss": 0.07423283159732819, |
| "num_tokens": 259055.0, |
| "reward": 0.2983750104904175, |
| "reward_std": 0.08031539618968964, |
| "rewards/reward_fn/mean": 0.2983750104904175, |
| "rewards/reward_fn/std": 0.08031539618968964, |
| "step": 90, |
| "step_time": 22.080729645000247 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 125.75, |
| "completions/mean_terminated_length": 125.75, |
| "completions/min_length": 105.0, |
| "completions/min_terminated_length": 105.0, |
| "entropy": 1.1015703678131104, |
| "epoch": 0.15166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.31640625, |
| "learning_rate": 2.7500000000000004e-06, |
| "loss": 0.033602241426706314, |
| "num_tokens": 261798.0, |
| "reward": 0.8031250238418579, |
| "reward_std": 0.3128456473350525, |
| "rewards/reward_fn/mean": 0.8031250238418579, |
| "rewards/reward_fn/std": 0.3128456473350525, |
| "step": 91, |
| "step_time": 23.45034698100062 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 145.0, |
| "completions/max_terminated_length": 145.0, |
| "completions/mean_length": 108.5, |
| "completions/mean_terminated_length": 108.5, |
| "completions/min_length": 83.0, |
| "completions/min_terminated_length": 83.0, |
| "entropy": 1.1214567422866821, |
| "epoch": 0.15333333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.421875, |
| "learning_rate": 2.7250000000000006e-06, |
| "loss": 0.02976590394973755, |
| "num_tokens": 264628.0, |
| "reward": 0.9728749990463257, |
| "reward_std": 0.015750011429190636, |
| "rewards/reward_fn/mean": 0.9728749990463257, |
| "rewards/reward_fn/std": 0.015750020742416382, |
| "step": 92, |
| "step_time": 22.66323952300081 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 146.0, |
| "completions/max_terminated_length": 146.0, |
| "completions/mean_length": 119.25, |
| "completions/mean_terminated_length": 119.25, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 1.1013864278793335, |
| "epoch": 0.155, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": 0.0016713386867195368, |
| "num_tokens": 267337.0, |
| "reward": 0.9772499799728394, |
| "reward_std": 0.015256185084581375, |
| "rewards/reward_fn/mean": 0.9772499799728394, |
| "rewards/reward_fn/std": 0.015256169252097607, |
| "step": 93, |
| "step_time": 22.21387937399777 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 142.0, |
| "completions/max_terminated_length": 142.0, |
| "completions/mean_length": 126.75, |
| "completions/mean_terminated_length": 126.75, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 1.1367976665496826, |
| "epoch": 0.15666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.333984375, |
| "learning_rate": 2.6750000000000002e-06, |
| "loss": 0.009300082921981812, |
| "num_tokens": 270352.0, |
| "reward": 0.48374998569488525, |
| "reward_std": 0.3209393620491028, |
| "rewards/reward_fn/mean": 0.48374998569488525, |
| "rewards/reward_fn/std": 0.3209393620491028, |
| "step": 94, |
| "step_time": 22.162854308000533 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 148.0, |
| "completions/max_terminated_length": 148.0, |
| "completions/mean_length": 123.75, |
| "completions/mean_terminated_length": 123.75, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.0440210103988647, |
| "epoch": 0.15833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42578125, |
| "learning_rate": 2.6500000000000005e-06, |
| "loss": -0.04094283655285835, |
| "num_tokens": 273143.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 95, |
| "step_time": 22.38568425199992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 160.0, |
| "completions/max_terminated_length": 160.0, |
| "completions/mean_length": 140.75, |
| "completions/mean_terminated_length": 140.75, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.1425111293792725, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.25390625, |
| "learning_rate": 2.6250000000000003e-06, |
| "loss": 0.15625080466270447, |
| "num_tokens": 275882.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.2921329736709595, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.2921329438686371, |
| "step": 96, |
| "step_time": 23.621867647998442 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 217.0, |
| "completions/max_terminated_length": 217.0, |
| "completions/mean_length": 165.5, |
| "completions/mean_terminated_length": 165.5, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 1.1441240310668945, |
| "epoch": 0.16166666666666665, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.283203125, |
| "learning_rate": 2.6e-06, |
| "loss": 0.12305271625518799, |
| "num_tokens": 278752.0, |
| "reward": 0.6497499942779541, |
| "reward_std": 0.3640195429325104, |
| "rewards/reward_fn/mean": 0.6497499942779541, |
| "rewards/reward_fn/std": 0.36401957273483276, |
| "step": 97, |
| "step_time": 32.032986541002174 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 135.0, |
| "completions/mean_terminated_length": 135.0, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.078916311264038, |
| "epoch": 0.16333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.302734375, |
| "learning_rate": 2.5750000000000003e-06, |
| "loss": -0.005260481499135494, |
| "num_tokens": 280888.0, |
| "reward": 0.6850000023841858, |
| "reward_std": 0.29560673236846924, |
| "rewards/reward_fn/mean": 0.6850000023841858, |
| "rewards/reward_fn/std": 0.29560670256614685, |
| "step": 98, |
| "step_time": 22.477214457998343 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 130.25, |
| "completions/mean_terminated_length": 130.25, |
| "completions/min_length": 102.0, |
| "completions/min_terminated_length": 102.0, |
| "entropy": 1.333487868309021, |
| "epoch": 0.165, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.384765625, |
| "learning_rate": 2.55e-06, |
| "loss": 0.1132018193602562, |
| "num_tokens": 283709.0, |
| "reward": 0.6368750333786011, |
| "reward_std": 0.35040080547332764, |
| "rewards/reward_fn/mean": 0.6368750333786011, |
| "rewards/reward_fn/std": 0.35040080547332764, |
| "step": 99, |
| "step_time": 22.616071093001665 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 141.0, |
| "completions/max_terminated_length": 141.0, |
| "completions/mean_length": 130.5, |
| "completions/mean_terminated_length": 130.5, |
| "completions/min_length": 115.0, |
| "completions/min_terminated_length": 115.0, |
| "entropy": 1.2107280492782593, |
| "epoch": 0.16666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.421875, |
| "learning_rate": 2.5250000000000004e-06, |
| "loss": -0.061634328216314316, |
| "num_tokens": 286963.0, |
| "reward": 0.3062500059604645, |
| "reward_std": 0.08250000327825546, |
| "rewards/reward_fn/mean": 0.3062500059604645, |
| "rewards/reward_fn/std": 0.08250000327825546, |
| "step": 100, |
| "step_time": 21.43296745800035 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 190.0, |
| "completions/max_terminated_length": 190.0, |
| "completions/mean_length": 167.5, |
| "completions/mean_terminated_length": 167.5, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.2127794027328491, |
| "epoch": 0.16833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2734375, |
| "learning_rate": 2.5e-06, |
| "loss": -0.01924452930688858, |
| "num_tokens": 289957.0, |
| "reward": 0.7856249809265137, |
| "reward_std": 0.30188557505607605, |
| "rewards/reward_fn/mean": 0.7856249809265137, |
| "rewards/reward_fn/std": 0.30188554525375366, |
| "step": 101, |
| "step_time": 28.20487242300078 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 247.0, |
| "completions/max_terminated_length": 247.0, |
| "completions/mean_length": 163.75, |
| "completions/mean_terminated_length": 163.75, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.3349406719207764, |
| "epoch": 0.17, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 2.475e-06, |
| "loss": 0.014912793412804604, |
| "num_tokens": 292740.0, |
| "reward": 0.437250018119812, |
| "reward_std": 0.21189679205417633, |
| "rewards/reward_fn/mean": 0.437250018119812, |
| "rewards/reward_fn/std": 0.21189679205417633, |
| "step": 102, |
| "step_time": 35.148211256997456 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 138.5, |
| "completions/mean_terminated_length": 138.5, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "entropy": 1.2255961894989014, |
| "epoch": 0.17166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.310546875, |
| "learning_rate": 2.4500000000000003e-06, |
| "loss": 0.09064196795225143, |
| "num_tokens": 295698.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.36373066902160645, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.36373066902160645, |
| "step": 103, |
| "step_time": 23.0402411569994 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 220.0, |
| "completions/max_terminated_length": 220.0, |
| "completions/mean_length": 183.5, |
| "completions/mean_terminated_length": 183.5, |
| "completions/min_length": 122.0, |
| "completions/min_terminated_length": 122.0, |
| "entropy": 1.368587613105774, |
| "epoch": 0.17333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.298828125, |
| "learning_rate": 2.425e-06, |
| "loss": -0.10342099517583847, |
| "num_tokens": 298992.0, |
| "reward": 0.47462502121925354, |
| "reward_std": 0.2910335958003998, |
| "rewards/reward_fn/mean": 0.47462502121925354, |
| "rewards/reward_fn/std": 0.29103362560272217, |
| "step": 104, |
| "step_time": 32.10494950900102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 172.0, |
| "completions/max_terminated_length": 172.0, |
| "completions/mean_length": 133.25, |
| "completions/mean_terminated_length": 133.25, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "entropy": 1.0861597061157227, |
| "epoch": 0.175, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.39453125, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": 0.1281687319278717, |
| "num_tokens": 301769.0, |
| "reward": 0.9650000333786011, |
| "reward_std": 0.02474874258041382, |
| "rewards/reward_fn/mean": 0.9650000333786011, |
| "rewards/reward_fn/std": 0.02474874258041382, |
| "step": 105, |
| "step_time": 25.204073330998654 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 173.0, |
| "completions/max_terminated_length": 173.0, |
| "completions/mean_length": 146.5, |
| "completions/mean_terminated_length": 146.5, |
| "completions/min_length": 111.0, |
| "completions/min_terminated_length": 111.0, |
| "entropy": 1.1364024877548218, |
| "epoch": 0.17666666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 2.375e-06, |
| "loss": 0.0, |
| "num_tokens": 304607.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 106, |
| "step_time": 25.721460639000725 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 159.25, |
| "completions/mean_terminated_length": 159.25, |
| "completions/min_length": 132.0, |
| "completions/min_terminated_length": 132.0, |
| "entropy": 1.2100403308868408, |
| "epoch": 0.17833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.248046875, |
| "learning_rate": 2.35e-06, |
| "loss": -0.1601293683052063, |
| "num_tokens": 307800.0, |
| "reward": 0.4881249964237213, |
| "reward_std": 0.3180236518383026, |
| "rewards/reward_fn/mean": 0.4881249964237213, |
| "rewards/reward_fn/std": 0.318023681640625, |
| "step": 107, |
| "step_time": 31.79101676999926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 183.0, |
| "completions/max_terminated_length": 183.0, |
| "completions/mean_length": 140.25, |
| "completions/mean_terminated_length": 140.25, |
| "completions/min_length": 117.0, |
| "completions/min_terminated_length": 117.0, |
| "entropy": 1.120354175567627, |
| "epoch": 0.18, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.345703125, |
| "learning_rate": 2.325e-06, |
| "loss": 0.05961640179157257, |
| "num_tokens": 310517.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.010103637352585793, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.010103637352585793, |
| "step": 108, |
| "step_time": 27.117513699002302 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 154.0, |
| "completions/max_terminated_length": 154.0, |
| "completions/mean_length": 147.25, |
| "completions/mean_terminated_length": 147.25, |
| "completions/min_length": 143.0, |
| "completions/min_terminated_length": 143.0, |
| "entropy": 1.1995975971221924, |
| "epoch": 0.18166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3046875, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.008527176454663277, |
| "num_tokens": 312970.0, |
| "reward": 0.6325000524520874, |
| "reward_std": 0.323947012424469, |
| "rewards/reward_fn/mean": 0.6325000524520874, |
| "rewards/reward_fn/std": 0.323947012424469, |
| "step": 109, |
| "step_time": 22.515986416998203 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 177.0, |
| "completions/max_terminated_length": 177.0, |
| "completions/mean_length": 139.0, |
| "completions/mean_terminated_length": 139.0, |
| "completions/min_length": 90.0, |
| "completions/min_terminated_length": 90.0, |
| "entropy": 1.2389236688613892, |
| "epoch": 0.18333333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.369140625, |
| "learning_rate": 2.2750000000000002e-06, |
| "loss": 0.23347245156764984, |
| "num_tokens": 315998.0, |
| "reward": 0.6587499976158142, |
| "reward_std": 0.35391557216644287, |
| "rewards/reward_fn/mean": 0.6587499976158142, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 110, |
| "step_time": 26.32129164900107 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 148.0, |
| "completions/max_terminated_length": 148.0, |
| "completions/mean_length": 131.75, |
| "completions/mean_terminated_length": 131.75, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.0297316312789917, |
| "epoch": 0.185, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.294921875, |
| "learning_rate": 2.25e-06, |
| "loss": -0.00834303256124258, |
| "num_tokens": 319005.0, |
| "reward": 0.9798749685287476, |
| "reward_std": 0.010451287031173706, |
| "rewards/reward_fn/mean": 0.9798749685287476, |
| "rewards/reward_fn/std": 0.010451287031173706, |
| "step": 111, |
| "step_time": 23.76591824099887 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 132.0, |
| "completions/max_terminated_length": 132.0, |
| "completions/mean_length": 117.0, |
| "completions/mean_terminated_length": 117.0, |
| "completions/min_length": 96.0, |
| "completions/min_terminated_length": 96.0, |
| "entropy": 0.9716232419013977, |
| "epoch": 0.18666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34765625, |
| "learning_rate": 2.2250000000000003e-06, |
| "loss": -0.04647402837872505, |
| "num_tokens": 322001.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 112, |
| "step_time": 21.865096360997995 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 194.0, |
| "completions/max_terminated_length": 194.0, |
| "completions/mean_length": 142.0, |
| "completions/mean_terminated_length": 142.0, |
| "completions/min_length": 103.0, |
| "completions/min_terminated_length": 103.0, |
| "entropy": 1.1111323833465576, |
| "epoch": 0.18833333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.28125, |
| "learning_rate": 2.2e-06, |
| "loss": -0.06792350113391876, |
| "num_tokens": 324989.0, |
| "reward": 0.7699999809265137, |
| "reward_std": 0.29849621653556824, |
| "rewards/reward_fn/mean": 0.7699999809265137, |
| "rewards/reward_fn/std": 0.29849621653556824, |
| "step": 113, |
| "step_time": 29.331891907997488 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 155.0, |
| "completions/max_terminated_length": 155.0, |
| "completions/mean_length": 122.25, |
| "completions/mean_terminated_length": 122.25, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.0778565406799316, |
| "epoch": 0.19, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.302734375, |
| "learning_rate": 2.1750000000000004e-06, |
| "loss": 0.041515663266181946, |
| "num_tokens": 327838.0, |
| "reward": 0.8206250071525574, |
| "reward_std": 0.3005298972129822, |
| "rewards/reward_fn/mean": 0.8206250071525574, |
| "rewards/reward_fn/std": 0.3005298972129822, |
| "step": 114, |
| "step_time": 24.199242210001103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 147.0, |
| "completions/max_terminated_length": 147.0, |
| "completions/mean_length": 135.25, |
| "completions/mean_terminated_length": 135.25, |
| "completions/min_length": 119.0, |
| "completions/min_terminated_length": 119.0, |
| "entropy": 1.2841877937316895, |
| "epoch": 0.19166666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37109375, |
| "learning_rate": 2.15e-06, |
| "loss": -0.036224182695150375, |
| "num_tokens": 330963.0, |
| "reward": 0.4881250262260437, |
| "reward_std": 0.318023681640625, |
| "rewards/reward_fn/mean": 0.4881250262260437, |
| "rewards/reward_fn/std": 0.318023681640625, |
| "step": 115, |
| "step_time": 22.69130362800206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 159.0, |
| "completions/max_terminated_length": 159.0, |
| "completions/mean_length": 144.25, |
| "completions/mean_terminated_length": 144.25, |
| "completions/min_length": 115.0, |
| "completions/min_terminated_length": 115.0, |
| "entropy": 1.1361361742019653, |
| "epoch": 0.19333333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 2.125e-06, |
| "loss": 0.0, |
| "num_tokens": 333908.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 116, |
| "step_time": 23.554109609001898 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 217.0, |
| "completions/max_terminated_length": 217.0, |
| "completions/mean_length": 172.75, |
| "completions/mean_terminated_length": 172.75, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "entropy": 1.189446210861206, |
| "epoch": 0.195, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": -0.01241180207580328, |
| "num_tokens": 336735.0, |
| "reward": 0.6412500143051147, |
| "reward_std": 0.35391560196876526, |
| "rewards/reward_fn/mean": 0.6412500143051147, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 117, |
| "step_time": 30.631830683996668 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 177.0, |
| "completions/max_terminated_length": 177.0, |
| "completions/mean_length": 137.75, |
| "completions/mean_terminated_length": 137.75, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 1.0830141305923462, |
| "epoch": 0.19666666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 2.075e-06, |
| "loss": 0.0, |
| "num_tokens": 338998.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 118, |
| "step_time": 24.650718585002323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 169.0, |
| "completions/max_terminated_length": 169.0, |
| "completions/mean_length": 146.0, |
| "completions/mean_terminated_length": 146.0, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.2152268886566162, |
| "epoch": 0.19833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.298828125, |
| "learning_rate": 2.05e-06, |
| "loss": 0.018085498362779617, |
| "num_tokens": 342026.0, |
| "reward": 0.8118749856948853, |
| "reward_std": 0.318023681640625, |
| "rewards/reward_fn/mean": 0.8118749856948853, |
| "rewards/reward_fn/std": 0.318023681640625, |
| "step": 119, |
| "step_time": 25.480774142000882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 180.0, |
| "completions/max_terminated_length": 180.0, |
| "completions/mean_length": 170.75, |
| "completions/mean_terminated_length": 170.75, |
| "completions/min_length": 159.0, |
| "completions/min_terminated_length": 159.0, |
| "entropy": 1.2765536308288574, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 2.025e-06, |
| "loss": -0.015368443913757801, |
| "num_tokens": 344997.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 120, |
| "step_time": 26.798823175002326 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 123.0, |
| "completions/mean_terminated_length": 123.0, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.0496407747268677, |
| "epoch": 0.20166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.375, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.10509224236011505, |
| "num_tokens": 347101.0, |
| "reward": 0.9562499523162842, |
| "reward_std": 0.017499983310699463, |
| "rewards/reward_fn/mean": 0.9562499523162842, |
| "rewards/reward_fn/std": 0.017499983310699463, |
| "step": 121, |
| "step_time": 21.680493171999842 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 129.75, |
| "completions/mean_terminated_length": 129.75, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.2729018926620483, |
| "epoch": 0.20333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.33984375, |
| "learning_rate": 1.975e-06, |
| "loss": 0.060011204332113266, |
| "num_tokens": 350008.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 122, |
| "step_time": 22.95859299299991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 120.0, |
| "completions/max_terminated_length": 120.0, |
| "completions/mean_length": 110.75, |
| "completions/mean_terminated_length": 110.75, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.0064460039138794, |
| "epoch": 0.205, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 1.9500000000000004e-06, |
| "loss": 0.0, |
| "num_tokens": 352667.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 123, |
| "step_time": 18.586940206998406 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 144.0, |
| "completions/max_terminated_length": 144.0, |
| "completions/mean_length": 117.25, |
| "completions/mean_terminated_length": 117.25, |
| "completions/min_length": 94.0, |
| "completions/min_terminated_length": 94.0, |
| "entropy": 1.0144492387771606, |
| "epoch": 0.20666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4375, |
| "learning_rate": 1.925e-06, |
| "loss": -0.09901498258113861, |
| "num_tokens": 355204.0, |
| "reward": 0.29750001430511475, |
| "reward_std": 0.07500000298023224, |
| "rewards/reward_fn/mean": 0.29750001430511475, |
| "rewards/reward_fn/std": 0.07500001043081284, |
| "step": 124, |
| "step_time": 21.89907346399923 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 149.0, |
| "completions/max_terminated_length": 149.0, |
| "completions/mean_length": 130.75, |
| "completions/mean_terminated_length": 130.75, |
| "completions/min_length": 118.0, |
| "completions/min_terminated_length": 118.0, |
| "entropy": 1.1170016527175903, |
| "epoch": 0.20833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.341796875, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.01803763210773468, |
| "num_tokens": 358015.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.010103637352585793, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.010103637352585793, |
| "step": 125, |
| "step_time": 22.859302955999738 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 148.0, |
| "completions/max_terminated_length": 148.0, |
| "completions/mean_length": 122.75, |
| "completions/mean_terminated_length": 122.75, |
| "completions/min_length": 90.0, |
| "completions/min_terminated_length": 90.0, |
| "entropy": 1.15077543258667, |
| "epoch": 0.21, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.33984375, |
| "learning_rate": 1.8750000000000003e-06, |
| "loss": 0.1318972259759903, |
| "num_tokens": 360842.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 126, |
| "step_time": 23.222802561998833 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 221.0, |
| "completions/max_terminated_length": 221.0, |
| "completions/mean_length": 207.5, |
| "completions/mean_terminated_length": 207.5, |
| "completions/min_length": 180.0, |
| "completions/min_terminated_length": 180.0, |
| "entropy": 1.2833231687545776, |
| "epoch": 0.21166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2197265625, |
| "learning_rate": 1.85e-06, |
| "loss": -0.06622515618801117, |
| "num_tokens": 364028.0, |
| "reward": 0.8159999847412109, |
| "reward_std": 0.297333687543869, |
| "rewards/reward_fn/mean": 0.8159999847412109, |
| "rewards/reward_fn/std": 0.297333687543869, |
| "step": 127, |
| "step_time": 33.3597098350001 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 155.0, |
| "completions/max_terminated_length": 155.0, |
| "completions/mean_length": 115.0, |
| "completions/mean_terminated_length": 115.0, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 0.9939418435096741, |
| "epoch": 0.21333333333333335, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 1.825e-06, |
| "loss": 0.0, |
| "num_tokens": 366980.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 128, |
| "step_time": 23.618680087998655 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 193.0, |
| "completions/max_terminated_length": 193.0, |
| "completions/mean_length": 144.0, |
| "completions/mean_terminated_length": 144.0, |
| "completions/min_length": 74.0, |
| "completions/min_terminated_length": 74.0, |
| "entropy": 1.1560587882995605, |
| "epoch": 0.215, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.318359375, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.16602951288223267, |
| "num_tokens": 369676.0, |
| "reward": 0.8041249513626099, |
| "reward_std": 0.31281474232673645, |
| "rewards/reward_fn/mean": 0.8041249513626099, |
| "rewards/reward_fn/std": 0.31281474232673645, |
| "step": 129, |
| "step_time": 28.466979378998076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 204.0, |
| "completions/max_terminated_length": 204.0, |
| "completions/mean_length": 149.0, |
| "completions/mean_terminated_length": 149.0, |
| "completions/min_length": 112.0, |
| "completions/min_terminated_length": 112.0, |
| "entropy": 1.2238303422927856, |
| "epoch": 0.21666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 1.7750000000000002e-06, |
| "loss": -0.006709250155836344, |
| "num_tokens": 372676.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 130, |
| "step_time": 30.414441388998966 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 195.0, |
| "completions/max_terminated_length": 195.0, |
| "completions/mean_length": 160.75, |
| "completions/mean_terminated_length": 160.75, |
| "completions/min_length": 129.0, |
| "completions/min_terminated_length": 129.0, |
| "entropy": 1.114032506942749, |
| "epoch": 0.21833333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 1.75e-06, |
| "loss": -0.0529935248196125, |
| "num_tokens": 375679.0, |
| "reward": 0.9772499799728394, |
| "reward_std": 0.015256185084581375, |
| "rewards/reward_fn/mean": 0.9772499799728394, |
| "rewards/reward_fn/std": 0.015256169252097607, |
| "step": 131, |
| "step_time": 28.930023082000844 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 187.0, |
| "completions/max_terminated_length": 187.0, |
| "completions/mean_length": 155.75, |
| "completions/mean_terminated_length": 155.75, |
| "completions/min_length": 130.0, |
| "completions/min_terminated_length": 130.0, |
| "entropy": 1.0404397249221802, |
| "epoch": 0.22, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.30078125, |
| "learning_rate": 1.725e-06, |
| "loss": -0.016852514818310738, |
| "num_tokens": 378594.0, |
| "reward": 0.32362502813339233, |
| "reward_std": 0.01381650846451521, |
| "rewards/reward_fn/mean": 0.32362502813339233, |
| "rewards/reward_fn/std": 0.01381650660187006, |
| "step": 132, |
| "step_time": 27.9692573909997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 158.0, |
| "completions/max_terminated_length": 158.0, |
| "completions/mean_length": 129.5, |
| "completions/mean_terminated_length": 129.5, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.1616339683532715, |
| "epoch": 0.22166666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3515625, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": -0.021115276962518692, |
| "num_tokens": 381292.0, |
| "reward": 0.34375, |
| "reward_std": 0.017500003799796104, |
| "rewards/reward_fn/mean": 0.34375, |
| "rewards/reward_fn/std": 0.017499998211860657, |
| "step": 133, |
| "step_time": 23.989496837002662 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 145.0, |
| "completions/max_terminated_length": 145.0, |
| "completions/mean_length": 137.75, |
| "completions/mean_terminated_length": 137.75, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "entropy": 1.1078652143478394, |
| "epoch": 0.22333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.267578125, |
| "learning_rate": 1.6750000000000003e-06, |
| "loss": 0.024493159726262093, |
| "num_tokens": 384255.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 134, |
| "step_time": 22.140087828000105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 130.75, |
| "completions/mean_terminated_length": 130.75, |
| "completions/min_length": 103.0, |
| "completions/min_terminated_length": 103.0, |
| "entropy": 1.1481397151947021, |
| "epoch": 0.225, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 1.6500000000000003e-06, |
| "loss": -0.05794854834675789, |
| "num_tokens": 386962.0, |
| "reward": 0.9728749990463257, |
| "reward_std": 0.015750011429190636, |
| "rewards/reward_fn/mean": 0.9728749990463257, |
| "rewards/reward_fn/std": 0.015750020742416382, |
| "step": 135, |
| "step_time": 22.68116860399823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 130.0, |
| "completions/max_terminated_length": 130.0, |
| "completions/mean_length": 124.0, |
| "completions/mean_terminated_length": 124.0, |
| "completions/min_length": 121.0, |
| "completions/min_terminated_length": 121.0, |
| "entropy": 1.2222754955291748, |
| "epoch": 0.22666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.330078125, |
| "learning_rate": 1.6250000000000001e-06, |
| "loss": -0.008061882108449936, |
| "num_tokens": 389566.0, |
| "reward": 0.8075000047683716, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.8075000047683716, |
| "rewards/reward_fn/std": 0.3149999976158142, |
| "step": 136, |
| "step_time": 19.605333135001274 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 137.0, |
| "completions/mean_terminated_length": 137.0, |
| "completions/min_length": 128.0, |
| "completions/min_terminated_length": 128.0, |
| "entropy": 1.1045880317687988, |
| "epoch": 0.22833333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": 0.0, |
| "num_tokens": 392290.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 137, |
| "step_time": 22.16470382499756 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 153.0, |
| "completions/max_terminated_length": 153.0, |
| "completions/mean_length": 129.5, |
| "completions/mean_terminated_length": 129.5, |
| "completions/min_length": 105.0, |
| "completions/min_terminated_length": 105.0, |
| "entropy": 1.1190224885940552, |
| "epoch": 0.23, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3671875, |
| "learning_rate": 1.5750000000000002e-06, |
| "loss": 0.07870414108037949, |
| "num_tokens": 394880.0, |
| "reward": 0.3062500059604645, |
| "reward_std": 0.08250001072883606, |
| "rewards/reward_fn/mean": 0.3062500059604645, |
| "rewards/reward_fn/std": 0.08250000327825546, |
| "step": 138, |
| "step_time": 22.143214091000118 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 223.0, |
| "completions/max_terminated_length": 223.0, |
| "completions/mean_length": 139.75, |
| "completions/mean_terminated_length": 139.75, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 1.0666494369506836, |
| "epoch": 0.23166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.373046875, |
| "learning_rate": 1.5500000000000002e-06, |
| "loss": -0.30225688219070435, |
| "num_tokens": 397699.0, |
| "reward": 0.47712501883506775, |
| "reward_std": 0.30046308040618896, |
| "rewards/reward_fn/mean": 0.47712501883506775, |
| "rewards/reward_fn/std": 0.30046311020851135, |
| "step": 139, |
| "step_time": 31.786553677000484 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 141.25, |
| "completions/mean_terminated_length": 141.25, |
| "completions/min_length": 134.0, |
| "completions/min_terminated_length": 134.0, |
| "entropy": 1.147861123085022, |
| "epoch": 0.23333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.333984375, |
| "learning_rate": 1.525e-06, |
| "loss": -0.03009037859737873, |
| "num_tokens": 400412.0, |
| "reward": 0.4802500009536743, |
| "reward_std": 0.32342347502708435, |
| "rewards/reward_fn/mean": 0.4802500009536743, |
| "rewards/reward_fn/std": 0.32342347502708435, |
| "step": 140, |
| "step_time": 22.017525466002553 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 172.0, |
| "completions/max_terminated_length": 172.0, |
| "completions/mean_length": 129.5, |
| "completions/mean_terminated_length": 129.5, |
| "completions/min_length": 81.0, |
| "completions/min_terminated_length": 81.0, |
| "entropy": 1.1366225481033325, |
| "epoch": 0.235, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.388671875, |
| "learning_rate": 1.5e-06, |
| "loss": 0.15167391300201416, |
| "num_tokens": 403278.0, |
| "reward": 0.8233749866485596, |
| "reward_std": 0.2794349789619446, |
| "rewards/reward_fn/mean": 0.8233749866485596, |
| "rewards/reward_fn/std": 0.2794349491596222, |
| "step": 141, |
| "step_time": 25.47716004799804 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 143.0, |
| "completions/max_terminated_length": 143.0, |
| "completions/mean_length": 133.5, |
| "completions/mean_terminated_length": 133.5, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "entropy": 1.071744680404663, |
| "epoch": 0.23666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3359375, |
| "learning_rate": 1.475e-06, |
| "loss": -0.029183873906731606, |
| "num_tokens": 405972.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.36373066902160645, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.36373066902160645, |
| "step": 142, |
| "step_time": 21.137350061002508 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 121.75, |
| "completions/mean_terminated_length": 121.75, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.0496933460235596, |
| "epoch": 0.23833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3671875, |
| "learning_rate": 1.45e-06, |
| "loss": 0.025060316547751427, |
| "num_tokens": 408671.0, |
| "reward": 0.484624981880188, |
| "reward_std": 0.32059407234191895, |
| "rewards/reward_fn/mean": 0.484624981880188, |
| "rewards/reward_fn/std": 0.32059407234191895, |
| "step": 143, |
| "step_time": 22.204842587998428 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 162.0, |
| "completions/max_terminated_length": 162.0, |
| "completions/mean_length": 124.25, |
| "completions/mean_terminated_length": 124.25, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.0704416036605835, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.357421875, |
| "learning_rate": 1.425e-06, |
| "loss": 0.1501941978931427, |
| "num_tokens": 411356.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 144, |
| "step_time": 23.816178979999677 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 136.0, |
| "completions/max_terminated_length": 136.0, |
| "completions/mean_length": 125.75, |
| "completions/mean_terminated_length": 125.75, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.1473332643508911, |
| "epoch": 0.24166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.423828125, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": 0.03730739653110504, |
| "num_tokens": 414443.0, |
| "reward": 0.918749988079071, |
| "reward_std": 0.07110730558633804, |
| "rewards/reward_fn/mean": 0.918749988079071, |
| "rewards/reward_fn/std": 0.07110730558633804, |
| "step": 145, |
| "step_time": 20.62395732100049 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 161.0, |
| "completions/max_terminated_length": 161.0, |
| "completions/mean_length": 133.0, |
| "completions/mean_terminated_length": 133.0, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.1500812768936157, |
| "epoch": 0.24333333333333335, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 1.3750000000000002e-06, |
| "loss": 0.0, |
| "num_tokens": 417223.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 146, |
| "step_time": 23.47268433299905 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 130.5, |
| "completions/mean_terminated_length": 130.5, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.165224313735962, |
| "epoch": 0.245, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.267578125, |
| "learning_rate": 1.3500000000000002e-06, |
| "loss": 0.08234935998916626, |
| "num_tokens": 420117.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 147, |
| "step_time": 22.157199847999436 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 127.25, |
| "completions/mean_terminated_length": 127.25, |
| "completions/min_length": 82.0, |
| "completions/min_terminated_length": 82.0, |
| "entropy": 1.0290559530258179, |
| "epoch": 0.24666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.32421875, |
| "learning_rate": 1.3250000000000002e-06, |
| "loss": -0.060243088752031326, |
| "num_tokens": 423030.0, |
| "reward": 0.4741249978542328, |
| "reward_std": 0.32808494567871094, |
| "rewards/reward_fn/mean": 0.4741249978542328, |
| "rewards/reward_fn/std": 0.32808491587638855, |
| "step": 148, |
| "step_time": 23.24264987100105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 184.0, |
| "completions/max_terminated_length": 184.0, |
| "completions/mean_length": 138.5, |
| "completions/mean_terminated_length": 138.5, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.1438663005828857, |
| "epoch": 0.24833333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 1.3e-06, |
| "loss": 0.11729130893945694, |
| "num_tokens": 425636.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 149, |
| "step_time": 27.72107682299975 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 203.0, |
| "completions/max_terminated_length": 203.0, |
| "completions/mean_length": 160.0, |
| "completions/mean_terminated_length": 160.0, |
| "completions/min_length": 128.0, |
| "completions/min_terminated_length": 128.0, |
| "entropy": 1.3133922815322876, |
| "epoch": 0.25, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.33203125, |
| "learning_rate": 1.275e-06, |
| "loss": 0.023503979668021202, |
| "num_tokens": 428360.0, |
| "reward": 0.49687498807907104, |
| "reward_std": 0.3128456175327301, |
| "rewards/reward_fn/mean": 0.49687498807907104, |
| "rewards/reward_fn/std": 0.3128456473350525, |
| "step": 150, |
| "step_time": 29.582455317002314 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 194.0, |
| "completions/max_terminated_length": 194.0, |
| "completions/mean_length": 140.75, |
| "completions/mean_terminated_length": 140.75, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.092279314994812, |
| "epoch": 0.25166666666666665, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.30078125, |
| "learning_rate": 1.25e-06, |
| "loss": 0.030243342742323875, |
| "num_tokens": 431399.0, |
| "reward": 0.3174999952316284, |
| "reward_std": 0.014288689009845257, |
| "rewards/reward_fn/mean": 0.3174999952316284, |
| "rewards/reward_fn/std": 0.014288689009845257, |
| "step": 151, |
| "step_time": 28.14050171800045 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 172.0, |
| "completions/max_terminated_length": 172.0, |
| "completions/mean_length": 141.0, |
| "completions/mean_terminated_length": 141.0, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.3847908973693848, |
| "epoch": 0.25333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.353515625, |
| "learning_rate": 1.2250000000000001e-06, |
| "loss": -0.11270540952682495, |
| "num_tokens": 434015.0, |
| "reward": 0.4881249964237213, |
| "reward_std": 0.2716260850429535, |
| "rewards/reward_fn/mean": 0.4881249964237213, |
| "rewards/reward_fn/std": 0.2716260850429535, |
| "step": 152, |
| "step_time": 25.02353806499741 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 180.0, |
| "completions/max_terminated_length": 180.0, |
| "completions/mean_length": 138.75, |
| "completions/mean_terminated_length": 138.75, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 1.2065187692642212, |
| "epoch": 0.255, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.291015625, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.15968742966651917, |
| "num_tokens": 436210.0, |
| "reward": 0.48375001549720764, |
| "reward_std": 0.27466264367103577, |
| "rewards/reward_fn/mean": 0.48375001549720764, |
| "rewards/reward_fn/std": 0.27466267347335815, |
| "step": 153, |
| "step_time": 25.63544118199934 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 135.0, |
| "completions/max_terminated_length": 135.0, |
| "completions/mean_length": 122.5, |
| "completions/mean_terminated_length": 122.5, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.0571459531784058, |
| "epoch": 0.25666666666666665, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.32421875, |
| "learning_rate": 1.175e-06, |
| "loss": -0.030262991786003113, |
| "num_tokens": 439040.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 154, |
| "step_time": 20.32930038700215 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 155.0, |
| "completions/max_terminated_length": 155.0, |
| "completions/mean_length": 135.5, |
| "completions/mean_terminated_length": 135.5, |
| "completions/min_length": 114.0, |
| "completions/min_terminated_length": 114.0, |
| "entropy": 1.005255937576294, |
| "epoch": 0.25833333333333336, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 1.1500000000000002e-06, |
| "loss": 0.0, |
| "num_tokens": 441942.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 155, |
| "step_time": 23.203776197999105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 156.0, |
| "completions/max_terminated_length": 156.0, |
| "completions/mean_length": 145.5, |
| "completions/mean_terminated_length": 145.5, |
| "completions/min_length": 133.0, |
| "completions/min_terminated_length": 133.0, |
| "entropy": 1.0427701473236084, |
| "epoch": 0.26, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 1.125e-06, |
| "loss": -0.035824455320835114, |
| "num_tokens": 444708.0, |
| "reward": 0.972000002861023, |
| "reward_std": 0.013999998569488525, |
| "rewards/reward_fn/mean": 0.972000002861023, |
| "rewards/reward_fn/std": 0.013999998569488525, |
| "step": 156, |
| "step_time": 23.09061835200191 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 202.0, |
| "completions/max_terminated_length": 202.0, |
| "completions/mean_length": 175.5, |
| "completions/mean_terminated_length": 175.5, |
| "completions/min_length": 155.0, |
| "completions/min_terminated_length": 155.0, |
| "entropy": 1.2932279109954834, |
| "epoch": 0.26166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.291015625, |
| "learning_rate": 1.1e-06, |
| "loss": 0.031955230981111526, |
| "num_tokens": 447946.0, |
| "reward": 0.5012500286102295, |
| "reward_std": 0.3096066117286682, |
| "rewards/reward_fn/mean": 0.5012500286102295, |
| "rewards/reward_fn/std": 0.3096066117286682, |
| "step": 157, |
| "step_time": 30.293572720001976 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 177.0, |
| "completions/max_terminated_length": 177.0, |
| "completions/mean_length": 136.0, |
| "completions/mean_terminated_length": 136.0, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.183039665222168, |
| "epoch": 0.2633333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.32421875, |
| "learning_rate": 1.075e-06, |
| "loss": -0.007350577507168055, |
| "num_tokens": 450582.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 158, |
| "step_time": 26.09575506499823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 132.25, |
| "completions/mean_terminated_length": 132.25, |
| "completions/min_length": 110.0, |
| "completions/min_terminated_length": 110.0, |
| "entropy": 1.0820621252059937, |
| "epoch": 0.265, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.306640625, |
| "learning_rate": 1.0500000000000001e-06, |
| "loss": -0.083620585501194, |
| "num_tokens": 453231.0, |
| "reward": 0.8162499666213989, |
| "reward_std": 0.3209393620491028, |
| "rewards/reward_fn/mean": 0.8162499666213989, |
| "rewards/reward_fn/std": 0.32093939185142517, |
| "step": 159, |
| "step_time": 22.651842983003007 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 185.0, |
| "completions/max_terminated_length": 185.0, |
| "completions/mean_length": 137.75, |
| "completions/mean_terminated_length": 137.75, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 1.2464767694473267, |
| "epoch": 0.26666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3125, |
| "learning_rate": 1.025e-06, |
| "loss": -0.023693839088082314, |
| "num_tokens": 456406.0, |
| "reward": 0.8162499666213989, |
| "reward_std": 0.32093939185142517, |
| "rewards/reward_fn/mean": 0.8162499666213989, |
| "rewards/reward_fn/std": 0.32093939185142517, |
| "step": 160, |
| "step_time": 27.780609208999522 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 201.0, |
| "completions/max_terminated_length": 201.0, |
| "completions/mean_length": 154.75, |
| "completions/mean_terminated_length": 154.75, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.316744327545166, |
| "epoch": 0.2683333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.326171875, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.1468632072210312, |
| "num_tokens": 459585.0, |
| "reward": 0.6587499976158142, |
| "reward_std": 0.3738343417644501, |
| "rewards/reward_fn/mean": 0.6587499976158142, |
| "rewards/reward_fn/std": 0.3738343119621277, |
| "step": 161, |
| "step_time": 29.671674831999553 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 132.0, |
| "completions/max_terminated_length": 132.0, |
| "completions/mean_length": 113.5, |
| "completions/mean_terminated_length": 113.5, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 1.0979673862457275, |
| "epoch": 0.27, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3828125, |
| "learning_rate": 9.750000000000002e-07, |
| "loss": -0.060441743582487106, |
| "num_tokens": 462603.0, |
| "reward": 0.32625001668930054, |
| "reward_std": 0.010103637352585793, |
| "rewards/reward_fn/mean": 0.32625001668930054, |
| "rewards/reward_fn/std": 0.010103637352585793, |
| "step": 162, |
| "step_time": 20.17059165000319 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 140.0, |
| "completions/max_terminated_length": 140.0, |
| "completions/mean_length": 127.25, |
| "completions/mean_terminated_length": 127.25, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.1192840337753296, |
| "epoch": 0.27166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 9.500000000000001e-07, |
| "loss": 0.06180550530552864, |
| "num_tokens": 465256.0, |
| "reward": 0.6543749570846558, |
| "reward_std": 0.3688516914844513, |
| "rewards/reward_fn/mean": 0.6543749570846558, |
| "rewards/reward_fn/std": 0.3688516914844513, |
| "step": 163, |
| "step_time": 20.585320939000667 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 146.0, |
| "completions/max_terminated_length": 146.0, |
| "completions/mean_length": 117.25, |
| "completions/mean_terminated_length": 117.25, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "entropy": 1.060671091079712, |
| "epoch": 0.2733333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 9.25e-07, |
| "loss": 0.0, |
| "num_tokens": 467961.0, |
| "reward": 0.33500000834465027, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.33500000834465027, |
| "rewards/reward_fn/std": 0.0, |
| "step": 164, |
| "step_time": 21.4030541309985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 162.0, |
| "completions/max_terminated_length": 162.0, |
| "completions/mean_length": 122.25, |
| "completions/mean_terminated_length": 122.25, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "entropy": 1.010462760925293, |
| "epoch": 0.275, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.23828125, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.12060170620679855, |
| "num_tokens": 470142.0, |
| "reward": 0.5056250095367432, |
| "reward_std": 0.30724838376045227, |
| "rewards/reward_fn/mean": 0.5056250095367432, |
| "rewards/reward_fn/std": 0.30724838376045227, |
| "step": 165, |
| "step_time": 23.291043189001357 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 237.0, |
| "completions/max_terminated_length": 237.0, |
| "completions/mean_length": 171.5, |
| "completions/mean_terminated_length": 171.5, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.3963382244110107, |
| "epoch": 0.27666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.291015625, |
| "learning_rate": 8.75e-07, |
| "loss": -0.0018487756606191397, |
| "num_tokens": 473312.0, |
| "reward": 0.6617499589920044, |
| "reward_std": 0.35750025510787964, |
| "rewards/reward_fn/mean": 0.6617499589920044, |
| "rewards/reward_fn/std": 0.35750025510787964, |
| "step": 166, |
| "step_time": 34.14518971199868 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 208.0, |
| "completions/max_terminated_length": 208.0, |
| "completions/mean_length": 163.0, |
| "completions/mean_terminated_length": 163.0, |
| "completions/min_length": 142.0, |
| "completions/min_terminated_length": 142.0, |
| "entropy": 1.246614933013916, |
| "epoch": 0.2783333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.314453125, |
| "learning_rate": 8.500000000000001e-07, |
| "loss": 0.13671796023845673, |
| "num_tokens": 476340.0, |
| "reward": 0.968500018119812, |
| "reward_std": 0.028719913214445114, |
| "rewards/reward_fn/mean": 0.968500018119812, |
| "rewards/reward_fn/std": 0.028719918802380562, |
| "step": 167, |
| "step_time": 30.15575728999829 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 149.0, |
| "completions/max_terminated_length": 149.0, |
| "completions/mean_length": 137.5, |
| "completions/mean_terminated_length": 137.5, |
| "completions/min_length": 129.0, |
| "completions/min_terminated_length": 129.0, |
| "entropy": 1.2037601470947266, |
| "epoch": 0.28, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.291015625, |
| "learning_rate": 8.250000000000001e-07, |
| "loss": 0.023372644558548927, |
| "num_tokens": 479566.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 168, |
| "step_time": 22.651985214000888 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 137.0, |
| "completions/max_terminated_length": 137.0, |
| "completions/mean_length": 119.5, |
| "completions/mean_terminated_length": 119.5, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 0.9957237243652344, |
| "epoch": 0.2816666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.337890625, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": -0.05584796518087387, |
| "num_tokens": 482692.0, |
| "reward": 0.3306249976158142, |
| "reward_std": 0.008750001899898052, |
| "rewards/reward_fn/mean": 0.3306249976158142, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 169, |
| "step_time": 20.86080199800199 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 140.0, |
| "completions/max_terminated_length": 140.0, |
| "completions/mean_length": 112.75, |
| "completions/mean_terminated_length": 112.75, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 1.0123518705368042, |
| "epoch": 0.2833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.322265625, |
| "learning_rate": 7.750000000000001e-07, |
| "loss": 0.013437695801258087, |
| "num_tokens": 485351.0, |
| "reward": 0.6324999928474426, |
| "reward_std": 0.34352341294288635, |
| "rewards/reward_fn/mean": 0.6324999928474426, |
| "rewards/reward_fn/std": 0.34352341294288635, |
| "step": 170, |
| "step_time": 20.707748422002624 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 174.0, |
| "completions/max_terminated_length": 174.0, |
| "completions/mean_length": 141.0, |
| "completions/mean_terminated_length": 141.0, |
| "completions/min_length": 113.0, |
| "completions/min_terminated_length": 113.0, |
| "entropy": 1.2882589101791382, |
| "epoch": 0.285, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.330078125, |
| "learning_rate": 7.5e-07, |
| "loss": -0.017979849129915237, |
| "num_tokens": 488211.0, |
| "reward": 0.7612500190734863, |
| "reward_std": 0.2913009524345398, |
| "rewards/reward_fn/mean": 0.7612500190734863, |
| "rewards/reward_fn/std": 0.2913009822368622, |
| "step": 171, |
| "step_time": 25.314323123999202 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 178.0, |
| "completions/max_terminated_length": 178.0, |
| "completions/mean_length": 137.0, |
| "completions/mean_terminated_length": 137.0, |
| "completions/min_length": 80.0, |
| "completions/min_terminated_length": 80.0, |
| "entropy": 1.2393285036087036, |
| "epoch": 0.2866666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.330078125, |
| "learning_rate": 7.25e-07, |
| "loss": 0.0024343333207070827, |
| "num_tokens": 491483.0, |
| "reward": 0.7942500114440918, |
| "reward_std": 0.30650001764297485, |
| "rewards/reward_fn/mean": 0.7942500114440918, |
| "rewards/reward_fn/std": 0.30649998784065247, |
| "step": 172, |
| "step_time": 26.337294665998343 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 167.0, |
| "completions/max_terminated_length": 167.0, |
| "completions/mean_length": 127.25, |
| "completions/mean_terminated_length": 127.25, |
| "completions/min_length": 98.0, |
| "completions/min_terminated_length": 98.0, |
| "entropy": 1.0118447542190552, |
| "epoch": 0.28833333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.326171875, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": -0.1553015410900116, |
| "num_tokens": 494264.0, |
| "reward": 0.34375, |
| "reward_std": 0.017500003799796104, |
| "rewards/reward_fn/mean": 0.34375, |
| "rewards/reward_fn/std": 0.017499998211860657, |
| "step": 173, |
| "step_time": 24.38981057300043 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 147.0, |
| "completions/max_terminated_length": 147.0, |
| "completions/mean_length": 132.25, |
| "completions/mean_terminated_length": 132.25, |
| "completions/min_length": 103.0, |
| "completions/min_terminated_length": 103.0, |
| "entropy": 1.3742601871490479, |
| "epoch": 0.29, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.328125, |
| "learning_rate": 6.750000000000001e-07, |
| "loss": 0.047720227390527725, |
| "num_tokens": 497373.0, |
| "reward": 0.8118749856948853, |
| "reward_std": 0.2953908145427704, |
| "rewards/reward_fn/mean": 0.8118749856948853, |
| "rewards/reward_fn/std": 0.2953908145427704, |
| "step": 174, |
| "step_time": 22.009925050002494 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 239.0, |
| "completions/max_terminated_length": 239.0, |
| "completions/mean_length": 158.5, |
| "completions/mean_terminated_length": 158.5, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.1549588441848755, |
| "epoch": 0.2916666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29296875, |
| "learning_rate": 6.5e-07, |
| "loss": 0.11253699660301208, |
| "num_tokens": 500407.0, |
| "reward": 0.9673749804496765, |
| "reward_std": 0.010765510611236095, |
| "rewards/reward_fn/mean": 0.9673749804496765, |
| "rewards/reward_fn/std": 0.010765498504042625, |
| "step": 175, |
| "step_time": 34.68952643800003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 176.0, |
| "completions/max_terminated_length": 176.0, |
| "completions/mean_length": 142.5, |
| "completions/mean_terminated_length": 142.5, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.1769404411315918, |
| "epoch": 0.29333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.33984375, |
| "learning_rate": 6.25e-07, |
| "loss": 0.11750662326812744, |
| "num_tokens": 503025.0, |
| "reward": 0.8075000047683716, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.8075000047683716, |
| "rewards/reward_fn/std": 0.3149999976158142, |
| "step": 176, |
| "step_time": 26.404072473997076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 151.0, |
| "completions/max_terminated_length": 151.0, |
| "completions/mean_length": 116.25, |
| "completions/mean_terminated_length": 116.25, |
| "completions/min_length": 86.0, |
| "completions/min_terminated_length": 86.0, |
| "entropy": 1.045095682144165, |
| "epoch": 0.295, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.392578125, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": -0.14939624071121216, |
| "num_tokens": 505870.0, |
| "reward": 0.484624981880188, |
| "reward_std": 0.32059407234191895, |
| "rewards/reward_fn/mean": 0.484624981880188, |
| "rewards/reward_fn/std": 0.32059407234191895, |
| "step": 177, |
| "step_time": 23.14946024099845 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 178.0, |
| "completions/max_terminated_length": 178.0, |
| "completions/mean_length": 138.25, |
| "completions/mean_terminated_length": 138.25, |
| "completions/min_length": 104.0, |
| "completions/min_terminated_length": 104.0, |
| "entropy": 1.0988469123840332, |
| "epoch": 0.2966666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.255859375, |
| "learning_rate": 5.750000000000001e-07, |
| "loss": 0.1224709302186966, |
| "num_tokens": 508679.0, |
| "reward": 0.3218750059604645, |
| "reward_std": 0.008750011213123798, |
| "rewards/reward_fn/mean": 0.3218750059604645, |
| "rewards/reward_fn/std": 0.008750006556510925, |
| "step": 178, |
| "step_time": 26.35802866800077 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 124.5, |
| "completions/mean_terminated_length": 124.5, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.0090521574020386, |
| "epoch": 0.29833333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3125, |
| "learning_rate": 5.5e-07, |
| "loss": 0.007333819754421711, |
| "num_tokens": 511573.0, |
| "reward": 0.48375001549720764, |
| "reward_std": 0.3209393620491028, |
| "rewards/reward_fn/mean": 0.48375001549720764, |
| "rewards/reward_fn/std": 0.3209393620491028, |
| "step": 179, |
| "step_time": 22.760440566002217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 143.0, |
| "completions/max_terminated_length": 143.0, |
| "completions/mean_length": 112.0, |
| "completions/mean_terminated_length": 112.0, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 1.2578190565109253, |
| "epoch": 0.3, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3515625, |
| "learning_rate": 5.250000000000001e-07, |
| "loss": 0.03865119442343712, |
| "num_tokens": 514181.0, |
| "reward": 0.6499999761581421, |
| "reward_std": 0.36373066902160645, |
| "rewards/reward_fn/mean": 0.6499999761581421, |
| "rewards/reward_fn/std": 0.36373066902160645, |
| "step": 180, |
| "step_time": 21.571868511000503 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 146.0, |
| "completions/max_terminated_length": 146.0, |
| "completions/mean_length": 124.75, |
| "completions/mean_terminated_length": 124.75, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.3287837505340576, |
| "epoch": 0.3016666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36328125, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": 0.05512908101081848, |
| "num_tokens": 517096.0, |
| "reward": 0.6412500143051147, |
| "reward_std": 0.35391560196876526, |
| "rewards/reward_fn/mean": 0.6412500143051147, |
| "rewards/reward_fn/std": 0.35391557216644287, |
| "step": 181, |
| "step_time": 21.837761578000936 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 152.0, |
| "completions/max_terminated_length": 152.0, |
| "completions/mean_length": 116.5, |
| "completions/mean_terminated_length": 116.5, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 1.0346094369888306, |
| "epoch": 0.30333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 4.7500000000000006e-07, |
| "loss": -0.10974273085594177, |
| "num_tokens": 520018.0, |
| "reward": 0.8136249780654907, |
| "reward_std": 0.29675617814064026, |
| "rewards/reward_fn/mean": 0.8136249780654907, |
| "rewards/reward_fn/std": 0.29675617814064026, |
| "step": 182, |
| "step_time": 22.461976462000166 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 138.0, |
| "completions/max_terminated_length": 138.0, |
| "completions/mean_length": 127.25, |
| "completions/mean_terminated_length": 127.25, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.0929187536239624, |
| "epoch": 0.305, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 4.5000000000000003e-07, |
| "loss": 0.042166367173194885, |
| "num_tokens": 522491.0, |
| "reward": 0.3062500059604645, |
| "reward_std": 0.057500001043081284, |
| "rewards/reward_fn/mean": 0.3062500059604645, |
| "rewards/reward_fn/std": 0.05750000849366188, |
| "step": 183, |
| "step_time": 20.686902885001473 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 122.0, |
| "completions/max_terminated_length": 122.0, |
| "completions/mean_length": 107.75, |
| "completions/mean_terminated_length": 107.75, |
| "completions/min_length": 95.0, |
| "completions/min_terminated_length": 95.0, |
| "entropy": 1.0409756898880005, |
| "epoch": 0.30666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36328125, |
| "learning_rate": 4.2500000000000006e-07, |
| "loss": 0.02638399973511696, |
| "num_tokens": 525490.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 184, |
| "step_time": 18.76558146499883 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 159.0, |
| "completions/max_terminated_length": 159.0, |
| "completions/mean_length": 136.25, |
| "completions/mean_terminated_length": 136.25, |
| "completions/min_length": 120.0, |
| "completions/min_terminated_length": 120.0, |
| "entropy": 1.1596969366073608, |
| "epoch": 0.30833333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3671875, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.023447571322321892, |
| "num_tokens": 528287.0, |
| "reward": 0.6631250381469727, |
| "reward_std": 0.3590344488620758, |
| "rewards/reward_fn/mean": 0.6631250381469727, |
| "rewards/reward_fn/std": 0.3590344488620758, |
| "step": 185, |
| "step_time": 23.50510012199811 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 168.0, |
| "completions/max_terminated_length": 168.0, |
| "completions/mean_length": 130.25, |
| "completions/mean_terminated_length": 130.25, |
| "completions/min_length": 107.0, |
| "completions/min_terminated_length": 107.0, |
| "entropy": 1.1854372024536133, |
| "epoch": 0.31, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.37890625, |
| "learning_rate": 3.75e-07, |
| "loss": -0.08003614097833633, |
| "num_tokens": 531576.0, |
| "reward": 0.9562499523162842, |
| "reward_std": 0.04165836051106453, |
| "rewards/reward_fn/mean": 0.9562499523162842, |
| "rewards/reward_fn/std": 0.041658345609903336, |
| "step": 186, |
| "step_time": 25.317642065001564 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 162.0, |
| "completions/max_terminated_length": 162.0, |
| "completions/mean_length": 132.0, |
| "completions/mean_terminated_length": 132.0, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.1301367282867432, |
| "epoch": 0.31166666666666665, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.31640625, |
| "learning_rate": 3.5000000000000004e-07, |
| "loss": -0.11360026895999908, |
| "num_tokens": 534248.0, |
| "reward": 0.4925000071525574, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.4925000071525574, |
| "rewards/reward_fn/std": 0.3149999678134918, |
| "step": 187, |
| "step_time": 24.889422044998355 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 170.0, |
| "completions/max_terminated_length": 170.0, |
| "completions/mean_length": 137.25, |
| "completions/mean_terminated_length": 137.25, |
| "completions/min_length": 84.0, |
| "completions/min_terminated_length": 84.0, |
| "entropy": 1.198452115058899, |
| "epoch": 0.31333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.298828125, |
| "learning_rate": 3.25e-07, |
| "loss": 0.06846462935209274, |
| "num_tokens": 536917.0, |
| "reward": 0.6456249952316284, |
| "reward_std": 0.35874998569488525, |
| "rewards/reward_fn/mean": 0.6456249952316284, |
| "rewards/reward_fn/std": 0.35874998569488525, |
| "step": 188, |
| "step_time": 24.68997424599729 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 249.0, |
| "completions/max_terminated_length": 249.0, |
| "completions/mean_length": 140.75, |
| "completions/mean_terminated_length": 140.75, |
| "completions/min_length": 99.0, |
| "completions/min_terminated_length": 99.0, |
| "entropy": 1.196223497390747, |
| "epoch": 0.315, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.375, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.2711350619792938, |
| "num_tokens": 539912.0, |
| "reward": 0.9668749570846558, |
| "reward_std": 0.01143370196223259, |
| "rewards/reward_fn/mean": 0.9668749570846558, |
| "rewards/reward_fn/std": 0.01143370196223259, |
| "step": 189, |
| "step_time": 35.739375315999496 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 173.0, |
| "completions/max_terminated_length": 173.0, |
| "completions/mean_length": 147.25, |
| "completions/mean_terminated_length": 147.25, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "entropy": 1.1819069385528564, |
| "epoch": 0.31666666666666665, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "learning_rate": 2.75e-07, |
| "loss": 0.0, |
| "num_tokens": 543321.0, |
| "reward": 0.9649999737739563, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9649999737739563, |
| "rewards/reward_fn/std": 0.0, |
| "step": 190, |
| "step_time": 26.15833759999805 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 174.0, |
| "completions/max_terminated_length": 174.0, |
| "completions/mean_length": 150.75, |
| "completions/mean_terminated_length": 150.75, |
| "completions/min_length": 129.0, |
| "completions/min_terminated_length": 129.0, |
| "entropy": 1.2341399192810059, |
| "epoch": 0.31833333333333336, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.365234375, |
| "learning_rate": 2.5000000000000004e-07, |
| "loss": 0.04181407764554024, |
| "num_tokens": 546288.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 191, |
| "step_time": 25.681773741998768 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 268.0, |
| "completions/max_terminated_length": 268.0, |
| "completions/mean_length": 165.5, |
| "completions/mean_terminated_length": 165.5, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 1.2476222515106201, |
| "epoch": 0.32, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.294921875, |
| "learning_rate": 2.2500000000000002e-07, |
| "loss": -0.02491738647222519, |
| "num_tokens": 549370.0, |
| "reward": 0.8023749589920044, |
| "reward_std": 0.3117331564426422, |
| "rewards/reward_fn/mean": 0.8023749589920044, |
| "rewards/reward_fn/std": 0.3117331564426422, |
| "step": 192, |
| "step_time": 38.91904668999996 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 135.0, |
| "completions/max_terminated_length": 135.0, |
| "completions/mean_length": 125.25, |
| "completions/mean_terminated_length": 125.25, |
| "completions/min_length": 115.0, |
| "completions/min_terminated_length": 115.0, |
| "entropy": 1.207331895828247, |
| "epoch": 0.32166666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29296875, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": -0.012936650775372982, |
| "num_tokens": 552399.0, |
| "reward": 0.9474999904632568, |
| "reward_std": 0.03500000759959221, |
| "rewards/reward_fn/mean": 0.9474999904632568, |
| "rewards/reward_fn/std": 0.034999996423721313, |
| "step": 193, |
| "step_time": 21.57561057500061 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 145.0, |
| "completions/max_terminated_length": 145.0, |
| "completions/mean_length": 125.0, |
| "completions/mean_terminated_length": 125.0, |
| "completions/min_length": 110.0, |
| "completions/min_terminated_length": 110.0, |
| "entropy": 1.171470046043396, |
| "epoch": 0.3233333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.375, |
| "learning_rate": 1.7500000000000002e-07, |
| "loss": 0.08878301829099655, |
| "num_tokens": 555579.0, |
| "reward": 0.6631250381469727, |
| "reward_std": 0.3590344190597534, |
| "rewards/reward_fn/mean": 0.6631250381469727, |
| "rewards/reward_fn/std": 0.3590344488620758, |
| "step": 194, |
| "step_time": 22.777231099997152 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 146.0, |
| "completions/max_terminated_length": 146.0, |
| "completions/mean_length": 125.25, |
| "completions/mean_terminated_length": 125.25, |
| "completions/min_length": 109.0, |
| "completions/min_terminated_length": 109.0, |
| "entropy": 1.1294211149215698, |
| "epoch": 0.325, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.359375, |
| "learning_rate": 1.5000000000000002e-07, |
| "loss": -0.06446050107479095, |
| "num_tokens": 558548.0, |
| "reward": 0.32712501287460327, |
| "reward_std": 0.015750011429190636, |
| "rewards/reward_fn/mean": 0.32712501287460327, |
| "rewards/reward_fn/std": 0.015750005841255188, |
| "step": 195, |
| "step_time": 22.614405715998146 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 137.0, |
| "completions/max_terminated_length": 137.0, |
| "completions/mean_length": 119.75, |
| "completions/mean_terminated_length": 119.75, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.0905414819717407, |
| "epoch": 0.32666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.36328125, |
| "learning_rate": 1.2500000000000002e-07, |
| "loss": 0.06022682413458824, |
| "num_tokens": 561567.0, |
| "reward": 0.9667500257492065, |
| "reward_std": 0.011608189903199673, |
| "rewards/reward_fn/mean": 0.9667500257492065, |
| "rewards/reward_fn/std": 0.011608189903199673, |
| "step": 196, |
| "step_time": 20.814484424998227 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 177.0, |
| "completions/max_terminated_length": 177.0, |
| "completions/mean_length": 145.5, |
| "completions/mean_terminated_length": 145.5, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "entropy": 1.0498154163360596, |
| "epoch": 0.3283333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2734375, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": 0.025765087455511093, |
| "num_tokens": 564581.0, |
| "reward": 0.8075000047683716, |
| "reward_std": 0.3149999976158142, |
| "rewards/reward_fn/mean": 0.8075000047683716, |
| "rewards/reward_fn/std": 0.3149999976158142, |
| "step": 197, |
| "step_time": 26.312665902001754 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 140.0, |
| "completions/max_terminated_length": 140.0, |
| "completions/mean_length": 119.75, |
| "completions/mean_terminated_length": 119.75, |
| "completions/min_length": 106.0, |
| "completions/min_terminated_length": 106.0, |
| "entropy": 1.0515799522399902, |
| "epoch": 0.33, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.326171875, |
| "learning_rate": 7.500000000000001e-08, |
| "loss": 0.010705118998885155, |
| "num_tokens": 567648.0, |
| "reward": 0.9772499799728394, |
| "reward_std": 0.015256163664162159, |
| "rewards/reward_fn/mean": 0.9772499799728394, |
| "rewards/reward_fn/std": 0.015256168320775032, |
| "step": 198, |
| "step_time": 21.37977665500148 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 170.0, |
| "completions/max_terminated_length": 170.0, |
| "completions/mean_length": 122.5, |
| "completions/mean_terminated_length": 122.5, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.0080755949020386, |
| "epoch": 0.33166666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.328125, |
| "learning_rate": 5.0000000000000004e-08, |
| "loss": -0.1868162900209427, |
| "num_tokens": 570646.0, |
| "reward": 0.34812501072883606, |
| "reward_std": 0.03880372270941734, |
| "rewards/reward_fn/mean": 0.34812501072883606, |
| "rewards/reward_fn/std": 0.03880372643470764, |
| "step": 199, |
| "step_time": 25.808188715000142 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 144.0, |
| "completions/max_terminated_length": 144.0, |
| "completions/mean_length": 118.25, |
| "completions/mean_terminated_length": 118.25, |
| "completions/min_length": 96.0, |
| "completions/min_terminated_length": 96.0, |
| "entropy": 1.0755765438079834, |
| "epoch": 0.3333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27734375, |
| "learning_rate": 2.5000000000000002e-08, |
| "loss": 0.09302064031362534, |
| "num_tokens": 573155.0, |
| "reward": 0.9693750143051147, |
| "reward_std": 0.00875002145767212, |
| "rewards/reward_fn/mean": 0.9693750143051147, |
| "rewards/reward_fn/std": 0.00875002145767212, |
| "step": 200, |
| "step_time": 21.597608337000565 |
| }, |
| { |
| "epoch": 0.3333333333333333, |
| "step": 200, |
| "total_flos": 0.0, |
| "train_loss": 0.0033340772934025154, |
| "train_runtime": 5010.8646, |
| "train_samples_per_second": 0.16, |
| "train_steps_per_second": 0.04 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 200, |
| "num_input_tokens_seen": 573155, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
