diff --git "a/checkpoint-350/trainer_state.json" "b/checkpoint-350/trainer_state.json"
--- "a/checkpoint-350/trainer_state.json"
+++ "b/checkpoint-350/trainer_state.json"
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.845528455284553,
+  "epoch": 8.536585365853659,
   "eval_steps": 500,
   "global_step": 350,
   "is_hyper_param_search": false,
@@ -16,26 +16,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 56.25,
-      "completions/mean_terminated_length": 56.25,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1208415031433105,
-      "epoch": 0.008130081300813009,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2371738702058792,
-      "kl": 1.1247546808590414e-05,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 60.10416793823242,
+      "completions/mean_terminated_length": 60.10416793823242,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2584454119205475,
+      "epoch": 0.024390243902439025,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09302648901939392,
+      "kl": 1.2248776783962967e-05,
       "learning_rate": 0.0,
-      "loss": 0.019987963140010834,
-      "num_tokens": 2250.0,
-      "reward": 0.7777429223060608,
-      "reward_std": 0.14680756628513336,
-      "rewards/true_env_reward_fn/mean": 0.7777429223060608,
-      "rewards/true_env_reward_fn/std": 0.14680756628513336,
+      "loss": -0.0423424206674099,
+      "num_tokens": 23029.0,
+      "reward": 0.5082165002822876,
+      "reward_std": 0.27811428904533386,
+      "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+      "rewards/true_env_reward_fn/std": 0.27811428904533386,
       "step": 1,
-      "step_time": 3.622400252999796
+      "step_time": 11.815711200999885
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 126.0,
-      "completions/max_terminated_length": 126.0,
-      "completions/mean_length": 72.875,
-      "completions/mean_terminated_length": 72.875,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 0.9768376648426056,
-      "epoch": 0.016260162601626018,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10255444794893265,
-      "kl": 9.209406016452704e-06,
-      "learning_rate": 2.1621621621621623e-08,
-      "loss": -0.16146813333034515,
-      "num_tokens": 8517.0,
-      "reward": 0.5471514463424683,
-      "reward_std": 0.19726651906967163,
-      "rewards/true_env_reward_fn/mean": 0.5471514463424683,
-      "rewards/true_env_reward_fn/std": 0.19726651906967163,
+      "completions/max_length": 161.0,
+      "completions/max_terminated_length": 161.0,
+      "completions/mean_length": 55.875,
+      "completions/mean_terminated_length": 55.875,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 1.3789870142936707,
+      "epoch": 0.04878048780487805,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11938872188329697,
+      "kl": 1.2672078355535632e-05,
+      "learning_rate": 2.4390243902439023e-08,
+      "loss": -0.11833255738019943,
+      "num_tokens": 57015.0,
+      "reward": 0.1327376663684845,
+      "reward_std": 0.241567462682724,
+      "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+      "rewards/true_env_reward_fn/std": 0.241567462682724,
       "step": 2,
-      "step_time": 5.979386726001394
+      "step_time": 13.493524850000085
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 67.125,
-      "completions/mean_terminated_length": 67.125,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.324017882347107,
-      "epoch": 0.024390243902439025,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13558730483055115,
-      "kl": 1.2776082257914823e-05,
-      "learning_rate": 4.3243243243243246e-08,
-      "loss": 0.008048340678215027,
-      "num_tokens": 11858.0,
-      "reward": 0.5399026870727539,
-      "reward_std": 0.04722921922802925,
-      "rewards/true_env_reward_fn/mean": 0.5399026870727539,
-      "rewards/true_env_reward_fn/std": 0.047229230403900146,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 63.79166793823242,
+      "completions/mean_terminated_length": 63.79166793823242,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "entropy": 1.315225213766098,
+      "epoch": 0.07317073170731707,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08519645780324936,
+      "kl": 1.2407871281538974e-05,
+      "learning_rate": 4.878048780487805e-08,
+      "loss": -0.03654177859425545,
+      "num_tokens": 86989.0,
+      "reward": 0.3152047097682953,
+      "reward_std": 0.3069385886192322,
+      "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+      "rewards/true_env_reward_fn/std": 0.30693864822387695,
       "step": 3,
-      "step_time": 3.6805073480009014
+      "step_time": 11.449303891999875
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 53.625,
-      "completions/mean_terminated_length": 53.625,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.0729783773422241,
-      "epoch": 0.032520325203252036,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23012493550777435,
-      "kl": 1.0804946214193478e-05,
-      "learning_rate": 6.486486486486487e-08,
-      "loss": 0.13091428577899933,
-      "num_tokens": 15379.0,
-      "reward": 0.4351762533187866,
-      "reward_std": 0.2320314645767212,
-      "rewards/true_env_reward_fn/mean": 0.4351762533187866,
-      "rewards/true_env_reward_fn/std": 0.2320314645767212,
+      "completions/max_length": 226.0,
+      "completions/max_terminated_length": 226.0,
+      "completions/mean_length": 77.20833587646484,
+      "completions/mean_terminated_length": 77.20833587646484,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.338063895702362,
+      "epoch": 0.0975609756097561,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08080132305622101,
+      "kl": 1.239982589140709e-05,
+      "learning_rate": 7.317073170731706e-08,
+      "loss": 0.053779490292072296,
+      "num_tokens": 112007.0,
+      "reward": 0.4893929362297058,
+      "reward_std": 0.28476035594940186,
+      "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+      "rewards/true_env_reward_fn/std": 0.28476035594940186,
       "step": 4,
-      "step_time": 3.421140036001816
+      "step_time": 18.835909622000145
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 57.25,
-      "completions/mean_terminated_length": 57.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.3000869154930115,
-      "epoch": 0.04065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22867721319198608,
-      "kl": 1.2170262834843015e-05,
-      "learning_rate": 8.648648648648649e-08,
-      "loss": 0.08851668983697891,
-      "num_tokens": 19401.0,
-      "reward": 0.4031979441642761,
-      "reward_std": 0.32033228874206543,
-      "rewards/true_env_reward_fn/mean": 0.4031979441642761,
-      "rewards/true_env_reward_fn/std": 0.32033228874206543,
+      "completions/max_length": 212.0,
+      "completions/max_terminated_length": 212.0,
+      "completions/mean_length": 67.41667175292969,
+      "completions/mean_terminated_length": 67.41667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3855182826519012,
+      "epoch": 0.12195121951219512,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08783729374408722,
+      "kl": 1.1660237760224845e-05,
+      "learning_rate": 9.75609756097561e-08,
+      "loss": -0.026884621009230614,
+      "num_tokens": 135883.0,
+      "reward": 0.48575252294540405,
+      "reward_std": 0.335994690656662,
+      "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+      "rewards/true_env_reward_fn/std": 0.335994690656662,
       "step": 5,
-      "step_time": 3.977350764000221
+      "step_time": 14.435845696000001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 58.5,
-      "completions/mean_terminated_length": 58.5,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.1719728112220764,
-      "epoch": 0.04878048780487805,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16202858090400696,
-      "kl": 1.228428209287813e-05,
-      "learning_rate": 1.0810810810810811e-07,
-      "loss": 0.1666201949119568,
-      "num_tokens": 21253.0,
-      "reward": 0.5827490091323853,
-      "reward_std": 0.27126544713974,
-      "rewards/true_env_reward_fn/mean": 0.5827490091323853,
-      "rewards/true_env_reward_fn/std": 0.27126544713974,
+      "completions/max_length": 164.0,
+      "completions/max_terminated_length": 164.0,
+      "completions/mean_length": 71.29167175292969,
+      "completions/mean_terminated_length": 71.29167175292969,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2962585091590881,
+      "epoch": 0.14634146341463414,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08510823547840118,
+      "kl": 1.241418908648484e-05,
+      "learning_rate": 1.219512195121951e-07,
+      "loss": -0.05353507027029991,
+      "num_tokens": 157537.0,
+      "reward": 0.47622889280319214,
+      "reward_std": 0.3605790138244629,
+      "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+      "rewards/true_env_reward_fn/std": 0.3605790138244629,
       "step": 6,
-      "step_time": 4.179320960000041
+      "step_time": 13.232063896999989
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.3112086653709412,
-      "epoch": 0.056910569105691054,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20938768982887268,
-      "kl": 1.306734156969469e-05,
-      "learning_rate": 1.2972972972972974e-07,
-      "loss": 0.04748187214136124,
-      "num_tokens": 25726.0,
-      "reward": 0.2716812491416931,
-      "reward_std": 0.29254475235939026,
-      "rewards/true_env_reward_fn/mean": 0.2716812491416931,
-      "rewards/true_env_reward_fn/std": 0.29254478216171265,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 69.45833587646484,
+      "completions/mean_terminated_length": 69.45833587646484,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.273663192987442,
+      "epoch": 0.17073170731707318,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0775279700756073,
+      "kl": 1.2900356978207128e-05,
+      "learning_rate": 1.4634146341463413e-07,
+      "loss": -0.010494321584701538,
+      "num_tokens": 179167.0,
+      "reward": 0.5062826871871948,
+      "reward_std": 0.18032674491405487,
+      "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+      "rewards/true_env_reward_fn/std": 0.18032673001289368,
       "step": 7,
-      "step_time": 3.438178512999002
+      "step_time": 9.810652986000036
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 78.5,
-      "completions/mean_terminated_length": 78.5,
-      "completions/min_length": 65.0,
-      "completions/min_terminated_length": 65.0,
-      "entropy": 1.2046615481376648,
-      "epoch": 0.06504065040650407,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.857898890506476e-05,
-      "kl": 1.3628536180476658e-05,
-      "learning_rate": 1.5135135135135135e-07,
-      "loss": 6.865971045044716e-07,
-      "num_tokens": 30126.0,
-      "reward": 0.49959999322891235,
-      "reward_std": 0.02822280302643776,
-      "rewards/true_env_reward_fn/mean": 0.49959999322891235,
-      "rewards/true_env_reward_fn/std": 0.02822280302643776,
+      "completions/max_length": 287.0,
+      "completions/max_terminated_length": 287.0,
+      "completions/mean_length": 65.54167175292969,
+      "completions/mean_terminated_length": 65.54167175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.255563884973526,
+      "epoch": 0.1951219512195122,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07464194297790527,
+      "kl": 1.1561841347429436e-05,
+      "learning_rate": 1.7073170731707317e-07,
+      "loss": 0.0830899029970169,
+      "num_tokens": 201865.0,
+      "reward": 0.38212963938713074,
+      "reward_std": 0.29894331097602844,
+      "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+      "rewards/true_env_reward_fn/std": 0.29894331097602844,
       "step": 8,
-      "step_time": 4.324984626000514
+      "step_time": 19.874756868999953
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.2111859917640686,
-      "epoch": 0.07317073170731707,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11552055925130844,
-      "kl": 1.0166647598452982e-05,
-      "learning_rate": 1.7297297297297298e-07,
-      "loss": -0.03883127495646477,
-      "num_tokens": 33332.0,
-      "reward": 0.571246862411499,
-      "reward_std": 0.2893567681312561,
-      "rewards/true_env_reward_fn/mean": 0.571246862411499,
-      "rewards/true_env_reward_fn/std": 0.2893567681312561,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 68.33333587646484,
+      "completions/mean_terminated_length": 68.33333587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2706169188022614,
+      "epoch": 0.21951219512195122,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.049192048609256744,
+      "kl": 1.157601468548819e-05,
+      "learning_rate": 1.951219512195122e-07,
+      "loss": 0.010864660143852234,
+      "num_tokens": 219953.0,
+      "reward": 0.6740004420280457,
+      "reward_std": 0.18809831142425537,
+      "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+      "rewards/true_env_reward_fn/std": 0.18809829652309418,
       "step": 9,
-      "step_time": 3.651253555999574
+      "step_time": 9.458149736999985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 58.125,
-      "completions/mean_terminated_length": 58.125,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.195803463459015,
-      "epoch": 0.08130081300813008,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14317302405834198,
-      "kl": 1.476421539337025e-05,
-      "learning_rate": 1.945945945945946e-07,
-      "loss": -0.034387920051813126,
-      "num_tokens": 36381.0,
-      "reward": 0.5593140125274658,
-      "reward_std": 0.42223072052001953,
-      "rewards/true_env_reward_fn/mean": 0.5593140125274658,
-      "rewards/true_env_reward_fn/std": 0.42223072052001953,
+      "completions/max_length": 148.0,
+      "completions/max_terminated_length": 148.0,
+      "completions/mean_length": 59.833335876464844,
+      "completions/mean_terminated_length": 59.833335876464844,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.1927059888839722,
+      "epoch": 0.24390243902439024,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.0561092346906662,
+      "kl": 1.0622535000948119e-05,
+      "learning_rate": 2.195121951219512e-07,
+      "loss": -0.02407176047563553,
+      "num_tokens": 244913.0,
+      "reward": 0.5113257169723511,
+      "reward_std": 0.32156965136528015,
+      "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+      "rewards/true_env_reward_fn/std": 0.32156962156295776,
       "step": 10,
-      "step_time": 3.8103441190014564
+      "step_time": 14.219840567000006
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 61.625,
-      "completions/mean_terminated_length": 61.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.389159917831421,
-      "epoch": 0.08943089430894309,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 65.47917175292969,
+      "completions/mean_terminated_length": 65.47917175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2782267928123474,
+      "epoch": 0.2682926829268293,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11816789954900742,
-      "kl": 1.0807365470100194e-05,
-      "learning_rate": 2.1621621621621622e-07,
-      "loss": 0.04055345058441162,
-      "num_tokens": 41830.0,
-      "reward": 0.12224999815225601,
-      "reward_std": 0.27913153171539307,
-      "rewards/true_env_reward_fn/mean": 0.12224999815225601,
-      "rewards/true_env_reward_fn/std": 0.27913153171539307,
+      "grad_norm": 0.05816411226987839,
+      "kl": 1.2071807759639341e-05,
+      "learning_rate": 2.439024390243902e-07,
+      "loss": 0.007693461142480373,
+      "num_tokens": 269080.0,
+      "reward": 0.37106746435165405,
+      "reward_std": 0.26608046889305115,
+      "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+      "rewards/true_env_reward_fn/std": 0.26608046889305115,
       "step": 11,
-      "step_time": 4.204996996000773
+      "step_time": 9.271131832999913
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.3866143822669983,
-      "epoch": 0.0975609756097561,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19537723064422607,
-      "kl": 1.5072046608111123e-05,
-      "learning_rate": 2.3783783783783783e-07,
-      "loss": -0.07735465466976166,
-      "num_tokens": 47047.0,
-      "reward": 0.3571999967098236,
-      "reward_std": 0.18295250833034515,
-      "rewards/true_env_reward_fn/mean": 0.3571999967098236,
-      "rewards/true_env_reward_fn/std": 0.18295250833034515,
+      "completions/max_length": 143.0,
+      "completions/max_terminated_length": 143.0,
+      "completions/mean_length": 67.9375,
+      "completions/mean_terminated_length": 67.9375,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3190773129463196,
+      "epoch": 0.2926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09140665084123611,
+      "kl": 1.2069708191120299e-05,
+      "learning_rate": 2.682926829268293e-07,
+      "loss": 0.07185906916856766,
+      "num_tokens": 291317.0,
+      "reward": 0.4376159906387329,
+      "reward_std": 0.27247554063796997,
+      "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+      "rewards/true_env_reward_fn/std": 0.27247554063796997,
       "step": 12,
-      "step_time": 4.775358541999594
+      "step_time": 12.184364300000084
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 55.75,
-      "completions/mean_terminated_length": 55.75,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.1633875966072083,
-      "epoch": 0.10569105691056911,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14852823317050934,
-      "kl": 1.4038786503078882e-05,
-      "learning_rate": 2.594594594594595e-07,
-      "loss": -0.04705440253019333,
-      "num_tokens": 51521.0,
-      "reward": 0.44465911388397217,
-      "reward_std": 0.15160730481147766,
-      "rewards/true_env_reward_fn/mean": 0.44465911388397217,
-      "rewards/true_env_reward_fn/std": 0.15160730481147766,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 66.54167175292969,
+      "completions/mean_terminated_length": 66.54167175292969,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.3555113077163696,
+      "epoch": 0.3170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08824986964464188,
+      "kl": 1.2127976788178785e-05,
+      "learning_rate": 2.9268292682926825e-07,
+      "loss": -0.0217185840010643,
+      "num_tokens": 313623.0,
+      "reward": 0.5092746615409851,
+      "reward_std": 0.3137436807155609,
+      "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+      "rewards/true_env_reward_fn/std": 0.3137436509132385,
       "step": 13,
-      "step_time": 4.072596639998665
+      "step_time": 10.720424850000086
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 60.375,
-      "completions/mean_terminated_length": 60.375,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.2392634153366089,
-      "epoch": 0.11382113821138211,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23616985976696014,
-      "kl": 1.3279905488161603e-05,
-      "learning_rate": 2.810810810810811e-07,
-      "loss": -0.021731968969106674,
-      "num_tokens": 55556.0,
-      "reward": 0.4130214750766754,
-      "reward_std": 0.43705809116363525,
-      "rewards/true_env_reward_fn/mean": 0.4130214750766754,
-      "rewards/true_env_reward_fn/std": 0.43705806136131287,
+      "completions/max_length": 188.0,
+      "completions/max_terminated_length": 188.0,
+      "completions/mean_length": 69.3125,
+      "completions/mean_terminated_length": 69.3125,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "entropy": 1.3283279240131378,
+      "epoch": 0.34146341463414637,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05055573210120201,
+      "kl": 1.3128182672517141e-05,
+      "learning_rate": 3.170731707317073e-07,
+      "loss": -0.024722743779420853,
+      "num_tokens": 339118.0,
+      "reward": 0.45545920729637146,
+      "reward_std": 0.18457132577896118,
+      "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+      "rewards/true_env_reward_fn/std": 0.18457134068012238,
       "step": 14,
-      "step_time": 3.7906999759998143
+      "step_time": 14.965493325000011
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 61.5,
-      "completions/mean_terminated_length": 61.5,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.1012902855873108,
-      "epoch": 0.12195121951219512,
+      "completions/max_length": 181.0,
+      "completions/max_terminated_length": 181.0,
+      "completions/mean_length": 66.45833587646484,
+      "completions/mean_terminated_length": 66.45833587646484,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.2629931271076202,
+      "epoch": 0.36585365853658536,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11504171043634415,
-      "kl": 1.1161580914631486e-05,
-      "learning_rate": 3.027027027027027e-07,
-      "loss": -0.03352496027946472,
-      "num_tokens": 58644.0,
-      "reward": 0.54444819688797,
-      "reward_std": 0.2691938281059265,
-      "rewards/true_env_reward_fn/mean": 0.54444819688797,
-      "rewards/true_env_reward_fn/std": 0.2691938281059265,
+      "grad_norm": 0.06967486441135406,
+      "kl": 1.1465989928183262e-05,
+      "learning_rate": 3.4146341463414634e-07,
+      "loss": 0.046319857239723206,
+      "num_tokens": 366364.0,
+      "reward": 0.4448578357696533,
+      "reward_std": 0.24966756999492645,
+      "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+      "rewards/true_env_reward_fn/std": 0.24966755509376526,
       "step": 15,
-      "step_time": 3.427628186998845
+      "step_time": 13.628413805999912
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 110.0,
-      "completions/max_terminated_length": 110.0,
-      "completions/mean_length": 66.125,
-      "completions/mean_terminated_length": 66.125,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.1984660625457764,
-      "epoch": 0.13008130081300814,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12652896344661713,
-      "kl": 1.292689512411016e-05,
-      "learning_rate": 3.243243243243243e-07,
-      "loss": -0.08787620067596436,
-      "num_tokens": 62361.0,
-      "reward": 0.46189582347869873,
-      "reward_std": 0.23188425600528717,
-      "rewards/true_env_reward_fn/mean": 0.46189582347869873,
-      "rewards/true_env_reward_fn/std": 0.23188428580760956,
+      "completions/max_length": 194.0,
+      "completions/max_terminated_length": 194.0,
+      "completions/mean_length": 69.04167175292969,
+      "completions/mean_terminated_length": 69.04167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2793545722961426,
+      "epoch": 0.3902439024390244,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.04725664108991623,
+      "kl": 1.1130929124192335e-05,
+      "learning_rate": 3.6585365853658536e-07,
+      "loss": 0.006799306720495224,
+      "num_tokens": 392926.0,
+      "reward": 0.414639949798584,
+      "reward_std": 0.2748004198074341,
+      "rewards/true_env_reward_fn/mean": 0.414639949798584,
+      "rewards/true_env_reward_fn/std": 0.2748004198074341,
       "step": 16,
-      "step_time": 4.776189491001787
+      "step_time": 14.229579036999894
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 58.25,
-      "completions/mean_terminated_length": 58.25,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1009634137153625,
-      "epoch": 0.13821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1329507976770401,
-      "kl": 1.1219160569453379e-05,
-      "learning_rate": 3.4594594594594597e-07,
-      "loss": 0.07219867408275604,
-      "num_tokens": 65899.0,
-      "reward": 0.65608811378479,
-      "reward_std": 0.2155800759792328,
-      "rewards/true_env_reward_fn/mean": 0.65608811378479,
-      "rewards/true_env_reward_fn/std": 0.21558009088039398,
+      "completions/max_length": 195.0,
+      "completions/max_terminated_length": 195.0,
+      "completions/mean_length": 76.4375,
+      "completions/mean_terminated_length": 76.4375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3106227219104767,
+      "epoch": 0.4146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06872504949569702,
+      "kl": 1.2065312830600305e-05,
+      "learning_rate": 3.902439024390244e-07,
+      "loss": 0.036527130752801895,
+      "num_tokens": 419219.0,
+      "reward": 0.49165210127830505,
+      "reward_std": 0.267509400844574,
+      "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+      "rewards/true_env_reward_fn/std": 0.267509400844574,
       "step": 17,
-      "step_time": 3.525365152998347
+      "step_time": 17.023353198999985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 51.5,
-      "completions/mean_terminated_length": 51.5,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.1671696901321411,
-      "epoch": 0.14634146341463414,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 71.72917175292969,
+      "completions/mean_terminated_length": 71.72917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3780030608177185,
+      "epoch": 0.43902439024390244,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15754961967468262,
-      "kl": 1.3107276572554838e-05,
-      "learning_rate": 3.6756756756756757e-07,
-      "loss": -0.016363894566893578,
-      "num_tokens": 70155.0,
-      "reward": 0.3013000190258026,
-      "reward_std": 0.2883487343788147,
-      "rewards/true_env_reward_fn/mean": 0.3013000190258026,
-      "rewards/true_env_reward_fn/std": 0.2883487641811371,
+      "grad_norm": 0.05453665927052498,
+      "kl": 1.2325858278927626e-05,
+      "learning_rate": 4.146341463414634e-07,
+      "loss": 0.01989848166704178,
+      "num_tokens": 442822.0,
+      "reward": 0.5288735032081604,
+      "reward_std": 0.2950553297996521,
+      "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+      "rewards/true_env_reward_fn/std": 0.2950552701950073,
       "step": 18,
-      "step_time": 4.143123763000403
+      "step_time": 11.965533113999868
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 111.0,
-      "completions/max_terminated_length": 111.0,
-      "completions/mean_length": 70.25,
-      "completions/mean_terminated_length": 70.25,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.266749083995819,
-      "epoch": 0.15447154471544716,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11665906757116318,
-      "kl": 1.2845626315538539e-05,
-      "learning_rate": 3.891891891891892e-07,
-      "loss": -0.11013027280569077,
-      "num_tokens": 73389.0,
-      "reward": 0.6058553457260132,
-      "reward_std": 0.11022671312093735,
-      "rewards/true_env_reward_fn/mean": 0.6058553457260132,
-      "rewards/true_env_reward_fn/std": 0.11022673547267914,
+      "completions/max_length": 106.0,
+      "completions/max_terminated_length": 106.0,
+      "completions/mean_length": 65.4375,
+      "completions/mean_terminated_length": 65.4375,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 1.3424750864505768,
+      "epoch": 0.4634146341463415,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09814280271530151,
+      "kl": 1.2686515219684225e-05,
+      "learning_rate": 4.390243902439024e-07,
+      "loss": 0.06940581649541855,
+      "num_tokens": 467275.0,
+      "reward": 0.5175753831863403,
+      "reward_std": 0.2811976969242096,
+      "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+      "rewards/true_env_reward_fn/std": 0.2811976969242096,
       "step": 19,
-      "step_time": 4.701202698999623
+      "step_time": 10.33812468799988
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 55.125,
-      "completions/mean_terminated_length": 55.125,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1111925840377808,
-      "epoch": 0.16260162601626016,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1429353505373001,
-      "kl": 8.694359621586045e-06,
-      "learning_rate": 4.108108108108108e-07,
-      "loss": -0.05066477507352829,
-      "num_tokens": 77594.0,
-      "reward": 0.4271581172943115,
-      "reward_std": 0.050101421773433685,
-      "rewards/true_env_reward_fn/mean": 0.4271581172943115,
-      "rewards/true_env_reward_fn/std": 0.05010143294930458,
+      "completions/max_length": 244.0,
+      "completions/max_terminated_length": 244.0,
+      "completions/mean_length": 65.10417175292969,
+      "completions/mean_terminated_length": 65.10417175292969,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.1681120097637177,
+      "epoch": 0.4878048780487805,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09474422037601471,
+      "kl": 1.2183225862827385e-05,
+      "learning_rate": 4.634146341463415e-07,
+      "loss": 0.05423373728990555,
+      "num_tokens": 494320.0,
+      "reward": 0.48628994822502136,
+      "reward_std": 0.25381213426589966,
+      "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+      "rewards/true_env_reward_fn/std": 0.25381216406822205,
       "step": 20,
-      "step_time": 3.220270914998764
+      "step_time": 17.317542748000164
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 46.0,
-      "completions/mean_terminated_length": 46.0,
-      "completions/min_length": 19.0,
-      "completions/min_terminated_length": 19.0,
-      "entropy": 1.4938308596611023,
-      "epoch": 0.17073170731707318,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16324248909950256,
-      "kl": 1.1220067335671047e-05,
-      "learning_rate": 4.3243243243243244e-07,
-      "loss": 0.023503631353378296,
-      "num_tokens": 83298.0,
-      "reward": 0.1186770498752594,
-      "reward_std": 0.16449356079101562,
-      "rewards/true_env_reward_fn/mean": 0.1186770498752594,
-      "rewards/true_env_reward_fn/std": 0.16449356079101562,
+      "completions/max_length": 157.0,
+      "completions/max_terminated_length": 157.0,
+      "completions/mean_length": 62.395835876464844,
+      "completions/mean_terminated_length": 62.395835876464844,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2504475116729736,
+      "epoch": 0.5121951219512195,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0819205492734909,
+      "kl": 1.0698822279664455e-05,
+      "learning_rate": 4.878048780487804e-07,
+      "loss": 0.05607657879590988,
+      "num_tokens": 518323.0,
+      "reward": 0.4693639278411865,
+      "reward_std": 0.32881346344947815,
+      "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+      "rewards/true_env_reward_fn/std": 0.32881346344947815,
       "step": 21,
-      "step_time": 3.451675898999383
+      "step_time": 12.20283881399996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2311039566993713,
-      "epoch": 0.17886178861788618,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23201963305473328,
-      "kl": 1.2657743809540989e-05,
-      "learning_rate": 4.54054054054054e-07,
-      "loss": 0.20273712277412415,
-      "num_tokens": 87825.0,
-      "reward": 0.3444172441959381,
-      "reward_std": 0.4508652687072754,
-      "rewards/true_env_reward_fn/mean": 0.3444172441959381,
-      "rewards/true_env_reward_fn/std": 0.450865238904953,
+      "completions/max_length": 144.0,
+      "completions/max_terminated_length": 144.0,
+      "completions/mean_length": 68.91667175292969,
+      "completions/mean_terminated_length": 68.91667175292969,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2199381291866302,
+      "epoch": 0.5365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06889473646879196,
+      "kl": 1.1745656820494332e-05,
+      "learning_rate": 5.121951219512195e-07,
+      "loss": -0.017973195761442184,
+      "num_tokens": 543591.0,
+      "reward": 0.49388420581817627,
+      "reward_std": 0.2952423393726349,
+      "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+      "rewards/true_env_reward_fn/std": 0.2952423095703125,
       "step": 22,
-      "step_time": 5.440214132999245
+      "step_time": 11.211206898000114
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 65.625,
+      "completions/mean_terminated_length": 65.625,
       "completions/min_length": 41.0,
       "completions/min_terminated_length": 41.0,
-      "entropy": 1.3744811415672302,
-      "epoch": 0.18699186991869918,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12073361128568649,
-      "kl": 1.356211032543797e-05,
-      "learning_rate": 4.7567567567567566e-07,
-      "loss": -0.06243758648633957,
-      "num_tokens": 92940.0,
-      "reward": 0.28657954931259155,
-      "reward_std": 0.19488918781280518,
-      "rewards/true_env_reward_fn/mean": 0.28657954931259155,
-      "rewards/true_env_reward_fn/std": 0.19488917291164398,
+      "entropy": 1.2588726878166199,
+      "epoch": 0.5609756097560976,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08144447952508926,
+      "kl": 1.2306870758038713e-05,
+      "learning_rate": 5.365853658536586e-07,
+      "loss": 0.02826106920838356,
+      "num_tokens": 567973.0,
+      "reward": 0.48142755031585693,
+      "reward_std": 0.26756224036216736,
+      "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+      "rewards/true_env_reward_fn/std": 0.26756221055984497,
       "step": 23,
-      "step_time": 4.085832714999924
+      "step_time": 10.428452587999914
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 100.0,
-      "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 68.875,
-      "completions/mean_terminated_length": 68.875,
-      "completions/min_length": 23.0,
-      "completions/min_terminated_length": 23.0,
-      "entropy": 1.3229535818099976,
-      "epoch": 0.1951219512195122,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.64079047460109e-05,
-      "kl": 1.1639681815722724e-05,
-      "learning_rate": 4.972972972972973e-07,
-      "loss": 5.819025545861223e-07,
-      "num_tokens": 94923.0,
-      "reward": 0.7253252267837524,
-      "reward_std": 0.046159788966178894,
-      "rewards/true_env_reward_fn/mean": 0.7253252267837524,
-      "rewards/true_env_reward_fn/std": 0.046159788966178894,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 59.5625,
+      "completions/mean_terminated_length": 59.5625,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.384379804134369,
+      "epoch": 0.5853658536585366,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11170398443937302,
+      "kl": 1.2296073691686615e-05,
+      "learning_rate": 5.609756097560975e-07,
+      "loss": 0.07271970808506012,
+      "num_tokens": 590248.0,
+      "reward": 0.38166365027427673,
+      "reward_std": 0.34809473156929016,
+      "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+      "rewards/true_env_reward_fn/std": 0.3480947017669678,
       "step": 24,
-      "step_time": 4.218084741001803
+      "step_time": 11.223491792000118
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 113.0,
-      "completions/max_terminated_length": 113.0,
-      "completions/mean_length": 76.375,
-      "completions/mean_terminated_length": 76.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.3325599431991577,
-      "epoch": 0.2032520325203252,
+      "completions/max_length": 123.0,
+      "completions/max_terminated_length": 123.0,
+      "completions/mean_length": 63.35416793823242,
+      "completions/mean_terminated_length": 63.35416793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.3013385236263275,
+      "epoch": 0.6097560975609756,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17998270690441132,
-      "kl": 1.4024041774973739e-05,
-      "learning_rate": 5.18918918918919e-07,
-      "loss": 0.13167564570903778,
-      "num_tokens": 99178.0,
-      "reward": 0.44252532720565796,
-      "reward_std": 0.1883804053068161,
-      "rewards/true_env_reward_fn/mean": 0.44252532720565796,
-      "rewards/true_env_reward_fn/std": 0.1883804053068161,
+      "grad_norm": 0.10069931298494339,
+      "kl": 1.2947949016961502e-05,
+      "learning_rate": 5.853658536585365e-07,
+      "loss": 0.033605337142944336,
+      "num_tokens": 615345.0,
+      "reward": 0.5046355724334717,
+      "reward_std": 0.2754679322242737,
+      "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+      "rewards/true_env_reward_fn/std": 0.2754679322242737,
       "step": 25,
-      "step_time": 4.84537445100068
+      "step_time": 10.92509102200006
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 64.375,
-      "completions/mean_terminated_length": 64.375,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.2788519263267517,
-      "epoch": 0.21138211382113822,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1275048851966858,
-      "kl": 1.1262640327913687e-05,
-      "learning_rate": 5.405405405405405e-07,
-      "loss": -0.010535649955272675,
-      "num_tokens": 102353.0,
-      "reward": 0.3852383494377136,
-      "reward_std": 0.2447713315486908,
-      "rewards/true_env_reward_fn/mean": 0.3852383494377136,
-      "rewards/true_env_reward_fn/std": 0.244771346449852,
+      "completions/max_length": 101.0,
+      "completions/max_terminated_length": 101.0,
+      "completions/mean_length": 61.41666793823242,
+      "completions/mean_terminated_length": 61.41666793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2652399837970734,
+      "epoch": 0.6341463414634146,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07595694065093994,
+      "kl": 1.151612354988174e-05,
+      "learning_rate": 6.097560975609756e-07,
+      "loss": 0.04607678949832916,
+      "num_tokens": 644749.0,
+      "reward": 0.3311978578567505,
+      "reward_std": 0.21527718007564545,
+      "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+      "rewards/true_env_reward_fn/std": 0.21527719497680664,
       "step": 26,
-      "step_time": 3.80895136899926
+      "step_time": 10.458724108999945
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 63.25,
-      "completions/mean_terminated_length": 63.25,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.0208025872707367,
-      "epoch": 0.21951219512195122,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14226751029491425,
-      "kl": 1.4639559594797902e-05,
-      "learning_rate": 5.621621621621622e-07,
-      "loss": -0.05629514902830124,
-      "num_tokens": 103867.0,
-      "reward": 0.8898874521255493,
-      "reward_std": 0.1414213478565216,
-      "rewards/true_env_reward_fn/mean": 0.8898874521255493,
-      "rewards/true_env_reward_fn/std": 0.1414213478565216,
+      "completions/max_length": 140.0,
+      "completions/max_terminated_length": 140.0,
+      "completions/mean_length": 71.25,
+      "completions/mean_terminated_length": 71.25,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.193794459104538,
+      "epoch": 0.6585365853658537,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07690244168043137,
+      "kl": 1.2164698546257569e-05,
+      "learning_rate": 6.341463414634146e-07,
+      "loss": 0.00818883627653122,
+      "num_tokens": 671153.0,
+      "reward": 0.3635203242301941,
+      "reward_std": 0.23849114775657654,
+      "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+      "rewards/true_env_reward_fn/std": 0.23849113285541534,
       "step": 27,
-      "step_time": 3.227140603999942
+      "step_time": 14.364785926000081
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 66.5,
-      "completions/mean_terminated_length": 66.5,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.278637707233429,
-      "epoch": 0.22764227642276422,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.6319210822694e-05,
-      "kl": 1.394796117892838e-05,
-      "learning_rate": 5.837837837837838e-07,
-      "loss": 6.984611218285863e-07,
-      "num_tokens": 108511.0,
-      "reward": 0.5384680032730103,
-      "reward_std": 0.06977442651987076,
-      "rewards/true_env_reward_fn/mean": 0.5384680032730103,
-      "rewards/true_env_reward_fn/std": 0.06977442651987076,
+      "completions/max_length": 111.0,
+      "completions/max_terminated_length": 111.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2883787751197815,
+      "epoch": 0.6829268292682927,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0902288407087326,
+      "kl": 1.1798915693361778e-05,
+      "learning_rate": 6.585365853658536e-07,
+      "loss": 0.038317371159791946,
+      "num_tokens": 697614.0,
+      "reward": 0.44166144728660583,
+      "reward_std": 0.25748196244239807,
+      "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+      "rewards/true_env_reward_fn/std": 0.25748199224472046,
       "step": 28,
-      "step_time": 3.3963304120006796
+      "step_time": 10.888908384999922
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 73.5,
-      "completions/mean_terminated_length": 73.5,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2547507286071777,
-      "epoch": 0.23577235772357724,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10846269875764847,
-      "kl": 1.5149432329053525e-05,
-      "learning_rate": 6.054054054054054e-07,
-      "loss": 0.004249647259712219,
-      "num_tokens": 111323.0,
-      "reward": 0.6256026029586792,
-      "reward_std": 0.350762277841568,
-      "rewards/true_env_reward_fn/mean": 0.6256026029586792,
-      "rewards/true_env_reward_fn/std": 0.350762277841568,
+      "completions/max_length": 238.0,
+      "completions/max_terminated_length": 238.0,
+      "completions/mean_length": 69.60417175292969,
+      "completions/mean_terminated_length": 69.60417175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3002805709838867,
+      "epoch": 0.7073170731707317,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07522639632225037,
+      "kl": 1.2230455695316778e-05,
+      "learning_rate": 6.829268292682927e-07,
+      "loss": 0.031045034527778625,
+      "num_tokens": 719187.0,
+      "reward": 0.5349087119102478,
+      "reward_std": 0.29909756779670715,
+      "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+      "rewards/true_env_reward_fn/std": 0.29909753799438477,
       "step": 29,
-      "step_time": 4.2103285969988065
+      "step_time": 15.510035302999995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.4687196612358093,
-      "epoch": 0.24390243902439024,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13451272249221802,
-      "kl": 1.3284446140460204e-05,
-      "learning_rate": 6.27027027027027e-07,
-      "loss": 0.05542291700839996,
-      "num_tokens": 115976.0,
-      "reward": 0.3901680111885071,
-      "reward_std": 0.2995865046977997,
-      "rewards/true_env_reward_fn/mean": 0.3901680111885071,
-      "rewards/true_env_reward_fn/std": 0.2995865046977997,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 189.0,
+      "completions/mean_length": 70.91667175292969,
+      "completions/mean_terminated_length": 70.91667175292969,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2718828916549683,
+      "epoch": 0.7317073170731707,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06776711344718933,
+      "kl": 1.2617916354429326e-05,
+      "learning_rate": 7.073170731707316e-07,
+      "loss": 0.09301326423883438,
+      "num_tokens": 744095.0,
+      "reward": 0.43472790718078613,
+      "reward_std": 0.3138841986656189,
+      "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+      "rewards/true_env_reward_fn/std": 0.3138841688632965,
       "step": 30,
-      "step_time": 3.5506420210003853
+      "step_time": 14.50245602599989
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 98.0,
-      "completions/max_terminated_length": 98.0,
-      "completions/mean_length": 76.875,
-      "completions/mean_terminated_length": 76.875,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2640270590782166,
-      "epoch": 0.25203252032520324,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18553969264030457,
-      "kl": 1.2505860468081664e-05,
-      "learning_rate": 6.486486486486486e-07,
-      "loss": -0.015417251735925674,
-      "num_tokens": 118471.0,
-      "reward": 0.6587758660316467,
-      "reward_std": 0.14417217671871185,
-      "rewards/true_env_reward_fn/mean": 0.6587758660316467,
-      "rewards/true_env_reward_fn/std": 0.14417219161987305,
+      "completions/max_length": 153.0,
+      "completions/max_terminated_length": 153.0,
+      "completions/mean_length": 69.77083587646484,
+      "completions/mean_terminated_length": 69.77083587646484,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2918945252895355,
+      "epoch": 0.7560975609756098,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08040682971477509,
+      "kl": 1.2672349157583085e-05,
+      "learning_rate": 7.317073170731707e-07,
+      "loss": 0.0367550291121006,
+      "num_tokens": 764612.0,
+      "reward": 0.5134401321411133,
+      "reward_std": 0.19073942303657532,
+      "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+      "rewards/true_env_reward_fn/std": 0.19073940813541412,
       "step": 31,
-      "step_time": 4.198089399002129
+      "step_time": 11.06186091799998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.5262224078178406,
-      "epoch": 0.2601626016260163,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2357814759016037,
-      "kl": 1.6242850506387185e-05,
-      "learning_rate": 6.702702702702702e-07,
-      "loss": 0.031210914254188538,
-      "num_tokens": 123923.0,
-      "reward": 0.0943702906370163,
-      "reward_std": 0.1497660130262375,
-      "rewards/true_env_reward_fn/mean": 0.0943702906370163,
-      "rewards/true_env_reward_fn/std": 0.1497660130262375,
+      "completions/max_length": 269.0,
+      "completions/max_terminated_length": 269.0,
+      "completions/mean_length": 71.79167175292969,
+      "completions/mean_terminated_length": 71.79167175292969,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.1679067015647888,
+      "epoch": 0.7804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0744430273771286,
+      "kl": 1.2661263326663175e-05,
+      "learning_rate": 7.560975609756097e-07,
+      "loss": 0.05885648727416992,
+      "num_tokens": 782058.0,
+      "reward": 0.5372593402862549,
+      "reward_std": 0.18350909650325775,
+      "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+      "rewards/true_env_reward_fn/std": 0.18350908160209656,
       "step": 32,
-      "step_time": 3.978757984001277
+      "step_time": 15.808748693000211
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "entropy": 0.9924907088279724,
-      "epoch": 0.2682926829268293,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20393438637256622,
-      "kl": 1.181096149593941e-05,
-      "learning_rate": 6.918918918918919e-07,
-      "loss": -0.0068489015102386475,
-      "num_tokens": 130831.0,
-      "reward": 0.20862048864364624,
-      "reward_std": 0.2418184131383896,
-      "rewards/true_env_reward_fn/mean": 0.20862048864364624,
-      "rewards/true_env_reward_fn/std": 0.2418184131383896,
+      "completions/max_length": 265.0,
+      "completions/max_terminated_length": 265.0,
+      "completions/mean_length": 76.79167175292969,
+      "completions/mean_terminated_length": 76.79167175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.1829756796360016,
+      "epoch": 0.8048780487804879,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.051698025315999985,
+      "kl": 1.0996191576850833e-05,
+      "learning_rate": 7.804878048780488e-07,
+      "loss": 0.010143717750906944,
+      "num_tokens": 810472.0,
+      "reward": 0.4369215667247772,
+      "reward_std": 0.30869919061660767,
+      "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+      "rewards/true_env_reward_fn/std": 0.30869919061660767,
       "step": 33,
-      "step_time": 4.237411461999727
+      "step_time": 24.20358999299981
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 184.0,
-      "completions/max_terminated_length": 184.0,
-      "completions/mean_length": 105.0,
-      "completions/mean_terminated_length": 105.0,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.4909422397613525,
-      "epoch": 0.2764227642276423,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.714608364040032e-05,
-      "kl": 1.3747331649938133e-05,
-      "learning_rate": 7.135135135135134e-07,
-      "loss": 6.856024583612452e-07,
-      "num_tokens": 138663.0,
-      "reward": 0.1821666657924652,
-      "reward_std": 0.2963036298751831,
-      "rewards/true_env_reward_fn/mean": 0.1821666657924652,
-      "rewards/true_env_reward_fn/std": 0.2963036298751831,
+      "completions/max_length": 137.0,
+      "completions/max_terminated_length": 137.0,
+      "completions/mean_length": 61.85416793823242,
+      "completions/mean_terminated_length": 61.85416793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2468958497047424,
+      "epoch": 0.8292682926829268,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09706687182188034,
+      "kl": 1.2097383432774222e-05,
+      "learning_rate": 8.048780487804878e-07,
+      "loss": 0.026558157056570053,
+      "num_tokens": 836713.0,
+      "reward": 0.3587157428264618,
+      "reward_std": 0.2754887044429779,
+      "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+      "rewards/true_env_reward_fn/std": 0.2754887044429779,
       "step": 34,
-      "step_time": 8.45711429900075
+      "step_time": 12.218407348999904
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 61.25,
-      "completions/mean_terminated_length": 61.25,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.0832659006118774,
-      "epoch": 0.2845528455284553,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1297609806060791,
-      "kl": 1.1829738923552213e-05,
-      "learning_rate": 7.351351351351351e-07,
-      "loss": -0.02754262089729309,
-      "num_tokens": 142361.0,
-      "reward": 0.4525124728679657,
-      "reward_std": 0.23157824575901031,
-      "rewards/true_env_reward_fn/mean": 0.4525124728679657,
-      "rewards/true_env_reward_fn/std": 0.2315782606601715,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 59.5625,
+      "completions/mean_terminated_length": 59.5625,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2368170320987701,
+      "epoch": 0.8536585365853658,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08964981138706207,
+      "kl": 1.3131634887031396e-05,
+      "learning_rate": 8.292682926829268e-07,
+      "loss": -0.01139204390347004,
+      "num_tokens": 860028.0,
+      "reward": 0.49109315872192383,
+      "reward_std": 0.20359393954277039,
+      "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+      "rewards/true_env_reward_fn/std": 0.20359393954277039,
       "step": 35,
-      "step_time": 3.564060039998367
+      "step_time": 9.66908789599995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 55.125,
-      "completions/mean_terminated_length": 55.125,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.0677781999111176,
-      "epoch": 0.2926829268292683,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14367543160915375,
-      "kl": 1.2750254427373875e-05,
-      "learning_rate": 7.567567567567568e-07,
-      "loss": -0.001130133867263794,
-      "num_tokens": 145294.0,
-      "reward": 0.6871603727340698,
-      "reward_std": 0.2714426517486572,
-      "rewards/true_env_reward_fn/mean": 0.6871603727340698,
-      "rewards/true_env_reward_fn/std": 0.2714426517486572,
+      "completions/max_length": 102.0,
+      "completions/max_terminated_length": 102.0,
+      "completions/mean_length": 66.02083587646484,
+      "completions/mean_terminated_length": 66.02083587646484,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.1611860394477844,
+      "epoch": 0.8780487804878049,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08851195871829987,
+      "kl": 1.2570341596074286e-05,
+      "learning_rate": 8.536585365853657e-07,
+      "loss": 0.021737128496170044,
+      "num_tokens": 883189.0,
+      "reward": 0.46058258414268494,
+      "reward_std": 0.2632383108139038,
+      "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+      "rewards/true_env_reward_fn/std": 0.2632383108139038,
       "step": 36,
-      "step_time": 3.6285808550001093
+      "step_time": 8.370980583999994
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 104.0,
-      "completions/max_terminated_length": 104.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.408882200717926,
-      "epoch": 0.3008130081300813,
+      "completions/max_length": 135.0,
+      "completions/max_terminated_length": 135.0,
+      "completions/mean_length": 75.58333587646484,
+      "completions/mean_terminated_length": 75.58333587646484,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "entropy": 1.37085822224617,
+      "epoch": 0.9024390243902439,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13866695761680603,
-      "kl": 1.3317891898623202e-05,
-      "learning_rate": 7.783783783783784e-07,
-      "loss": 0.058712199330329895,
-      "num_tokens": 148747.0,
-      "reward": 0.638524055480957,
-      "reward_std": 0.380489706993103,
-      "rewards/true_env_reward_fn/mean": 0.638524055480957,
-      "rewards/true_env_reward_fn/std": 0.3804897367954254,
+      "grad_norm": 0.05852028727531433,
+      "kl": 1.2957561011717189e-05,
+      "learning_rate": 8.780487804878048e-07,
+      "loss": -0.024281952530145645,
+      "num_tokens": 906801.0,
+      "reward": 0.5022324323654175,
+      "reward_std": 0.11637427657842636,
+      "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+      "rewards/true_env_reward_fn/std": 0.11637428402900696,
       "step": 37,
-      "step_time": 4.57648780099953
+      "step_time": 10.285125336999727
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.3680316805839539,
-      "epoch": 0.3089430894308943,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13230997323989868,
-      "kl": 1.1831724805233534e-05,
-      "learning_rate": 8e-07,
-      "loss": -0.06476183235645294,
-      "num_tokens": 152794.0,
-      "reward": 0.47908467054367065,
-      "reward_std": 0.18681679666042328,
-      "rewards/true_env_reward_fn/mean": 0.47908467054367065,
-      "rewards/true_env_reward_fn/std": 0.18681679666042328,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 65.14583587646484,
+      "completions/mean_terminated_length": 65.14583587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2760809361934662,
+      "epoch": 0.926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09105321019887924,
+      "kl": 1.3129126955391257e-05,
+      "learning_rate": 9.024390243902439e-07,
+      "loss": -0.011838603764772415,
+      "num_tokens": 929536.0,
+      "reward": 0.49639374017715454,
+      "reward_std": 0.32166802883148193,
+      "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+      "rewards/true_env_reward_fn/std": 0.32166802883148193,
       "step": 38,
-      "step_time": 3.627890882000429
+      "step_time": 12.449738128000035
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 56.875,
-      "completions/mean_terminated_length": 56.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.3124344944953918,
-      "epoch": 0.3170731707317073,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20907950401306152,
-      "kl": 1.4425510926230345e-05,
-      "learning_rate": 7.999820918660971e-07,
-      "loss": -0.014620006084442139,
-      "num_tokens": 157337.0,
-      "reward": 0.4882892966270447,
-      "reward_std": 0.28137314319610596,
-      "rewards/true_env_reward_fn/mean": 0.4882892966270447,
-      "rewards/true_env_reward_fn/std": 0.28137317299842834,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 72.08333587646484,
+      "completions/mean_terminated_length": 72.08333587646484,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.2545586228370667,
+      "epoch": 0.9512195121951219,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06919296830892563,
+      "kl": 1.459557256566768e-05,
+      "learning_rate": 9.26829268292683e-07,
+      "loss": 0.021831180900335312,
+      "num_tokens": 950388.0,
+      "reward": 0.4879913330078125,
+      "reward_std": 0.24854585528373718,
+      "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+      "rewards/true_env_reward_fn/std": 0.24854585528373718,
       "step": 39,
-      "step_time": 3.5362214279994078
+      "step_time": 10.279209028999958
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 51.625,
-      "completions/mean_terminated_length": 51.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 0.9928885996341705,
-      "epoch": 0.3252032520325203,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.641438762424514e-05,
-      "kl": 1.1731265658454504e-05,
-      "learning_rate": 7.99928369067895e-07,
-      "loss": 5.910313234380737e-07,
-      "num_tokens": 160166.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/max_length": 179.0,
+      "completions/max_terminated_length": 179.0,
+      "completions/mean_length": 74.20833587646484,
+      "completions/mean_terminated_length": 74.20833587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2255937159061432,
+      "epoch": 0.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06352153420448303,
+      "kl": 1.2041192348988261e-05,
+      "learning_rate": 9.512195121951218e-07,
+      "loss": -0.013997981324791908,
+      "num_tokens": 981254.0,
+      "reward": 0.39802420139312744,
+      "reward_std": 0.20212584733963013,
+      "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+      "rewards/true_env_reward_fn/std": 0.20212584733963013,
       "step": 40,
-      "step_time": 3.1957039770022675
+      "step_time": 13.58010066599968
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 54.75,
-      "completions/mean_terminated_length": 54.75,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.2997828722000122,
-      "epoch": 0.3333333333333333,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21271590888500214,
-      "kl": 1.3209032658778597e-05,
-      "learning_rate": 7.99838836415769e-07,
-      "loss": 0.033298641443252563,
-      "num_tokens": 165884.0,
-      "reward": 0.2860966920852661,
-      "reward_std": 0.2721884846687317,
-      "rewards/true_env_reward_fn/mean": 0.2860966920852661,
-      "rewards/true_env_reward_fn/std": 0.2721884846687317,
+      "completions/max_length": 139.0,
+      "completions/max_terminated_length": 139.0,
+      "completions/mean_length": 75.04167175292969,
+      "completions/mean_terminated_length": 75.04167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2703719735145569,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.045169439166784286,
+      "kl": 1.1270850109212915e-05,
+      "learning_rate": 9.756097560975609e-07,
+      "loss": -0.010194316506385803,
+      "num_tokens": 1009968.0,
+      "reward": 0.4517599940299988,
+      "reward_std": 0.11791092902421951,
+      "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+      "rewards/true_env_reward_fn/std": 0.11791091412305832,
       "step": 41,
-      "step_time": 3.6851942720004445
+      "step_time": 10.35077203700007
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 48.875,
-      "completions/mean_terminated_length": 48.875,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.4380556344985962,
-      "epoch": 0.34146341463414637,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.17314757406711578,
-      "kl": 9.354779194836738e-06,
-      "learning_rate": 7.997135019265325e-07,
-      "loss": 0.08398272097110748,
-      "num_tokens": 172067.0,
-      "reward": -0.003943998366594315,
-      "reward_std": 0.13122709095478058,
-      "rewards/true_env_reward_fn/mean": -0.003943998366594315,
-      "rewards/true_env_reward_fn/std": 0.13122709095478058,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 64.33333587646484,
+      "completions/mean_terminated_length": 64.33333587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.329576164484024,
+      "epoch": 1.024390243902439,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08522730320692062,
+      "kl": 1.4469044799625408e-05,
+      "learning_rate": 1e-06,
+      "loss": -0.00014946190640330315,
+      "num_tokens": 1039032.0,
+      "reward": 0.33548423647880554,
+      "reward_std": 0.22271563112735748,
+      "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+      "rewards/true_env_reward_fn/std": 0.22271563112735748,
       "step": 42,
-      "step_time": 3.545334507000007
+      "step_time": 10.548370664999993
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.0,
-      "completions/max_terminated_length": 123.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 71.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.163844645023346,
-      "epoch": 0.34959349593495936,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15747681260108948,
-      "kl": 1.3550960375141585e-05,
-      "learning_rate": 7.995523768227198e-07,
-      "loss": 0.05901219695806503,
-      "num_tokens": 176427.0,
-      "reward": 0.3297747075557709,
-      "reward_std": 0.4647028148174286,
-      "rewards/true_env_reward_fn/mean": 0.3297747075557709,
-      "rewards/true_env_reward_fn/std": 0.464702844619751,
+      "completions/max_length": 372.0,
+      "completions/max_terminated_length": 372.0,
+      "completions/mean_length": 70.02083587646484,
+      "completions/mean_terminated_length": 70.02083587646484,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.2357364892959595,
+      "epoch": 1.048780487804878,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07030358910560608,
+      "kl": 1.3562755839302554e-05,
+      "learning_rate": 9.999818789066163e-07,
+      "loss": -0.02616041898727417,
+      "num_tokens": 1060833.0,
+      "reward": 0.5167371034622192,
+      "reward_std": 0.24280032515525818,
+      "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+      "rewards/true_env_reward_fn/std": 0.24280032515525818,
       "step": 43,
-      "step_time": 5.4708715960005065
+      "step_time": 24.089396637999698
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.3323996663093567,
-      "epoch": 0.35772357723577236,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15227818489074707,
-      "kl": 1.1237668786634458e-05,
-      "learning_rate": 7.993554755315805e-07,
-      "loss": 0.0660967156291008,
-      "num_tokens": 181912.0,
-      "reward": 0.22226500511169434,
-      "reward_std": 0.2765512466430664,
-      "rewards/true_env_reward_fn/mean": 0.22226500511169434,
-      "rewards/true_env_reward_fn/std": 0.2765512466430664,
+      "completions/max_length": 234.0,
+      "completions/max_terminated_length": 234.0,
+      "completions/mean_length": 77.47917175292969,
+      "completions/mean_terminated_length": 77.47917175292969,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 1.1693778038024902,
+      "epoch": 1.0731707317073171,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07017157226800919,
+      "kl": 1.332453393843025e-05,
+      "learning_rate": 9.999275169399612e-07,
+      "loss": -0.006466507911682129,
+      "num_tokens": 1088648.0,
+      "reward": 0.4498252272605896,
+      "reward_std": 0.21398545801639557,
+      "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+      "rewards/true_env_reward_fn/std": 0.21398545801639557,
       "step": 44,
-      "step_time": 3.940563359999942
+      "step_time": 19.39071501599983
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 56.375,
-      "completions/mean_terminated_length": 56.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.241302490234375,
-      "epoch": 0.36585365853658536,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1711702048778534,
-      "kl": 1.1479866316221887e-05,
-      "learning_rate": 7.991228156837879e-07,
-      "loss": 0.0959811806678772,
-      "num_tokens": 186099.0,
-      "reward": 0.4569639563560486,
-      "reward_std": 0.356449693441391,
-      "rewards/true_env_reward_fn/mean": 0.4569639563560486,
-      "rewards/true_env_reward_fn/std": 0.356449693441391,
+      "completions/max_length": 186.0,
+      "completions/max_terminated_length": 186.0,
+      "completions/mean_length": 72.16667175292969,
+      "completions/mean_terminated_length": 72.16667175292969,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3268415927886963,
+      "epoch": 1.0975609756097562,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06632921099662781,
+      "kl": 1.4458733630817733e-05,
+      "learning_rate": 9.99836918040428e-07,
+      "loss": -0.03534461930394173,
+      "num_tokens": 1117096.0,
+      "reward": 0.4053138196468353,
+      "reward_std": 0.21476909518241882,
+      "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+      "rewards/true_env_reward_fn/std": 0.21476909518241882,
       "step": 45,
-      "step_time": 3.947248132999448
+      "step_time": 13.893569495999827
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 66.625,
-      "completions/mean_terminated_length": 66.625,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.5153677463531494,
-      "epoch": 0.37398373983739835,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22329360246658325,
-      "kl": 1.3615897842100821e-05,
-      "learning_rate": 7.988544181118608e-07,
-      "loss": 0.07407481223344803,
-      "num_tokens": 192056.0,
-      "reward": 0.2950569987297058,
-      "reward_std": 0.2872281074523926,
-      "rewards/true_env_reward_fn/mean": 0.2950569987297058,
-      "rewards/true_env_reward_fn/std": 0.28722813725471497,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 70.16667175292969,
+      "completions/mean_terminated_length": 70.16667175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2670875787734985,
+      "epoch": 1.1219512195121952,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08321154117584229,
+      "kl": 1.4837954950053245e-05,
+      "learning_rate": 9.997100887750215e-07,
+      "loss": -0.039235007017850876,
+      "num_tokens": 1136480.0,
+      "reward": 0.48141974210739136,
+      "reward_std": 0.2837103307247162,
+      "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+      "rewards/true_env_reward_fn/std": 0.2837103009223938,
       "step": 46,
-      "step_time": 4.1211709569997765
+      "step_time": 10.50698806499986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.105223298072815,
-      "epoch": 0.3821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1306377500295639,
-      "kl": 1.2826244528696407e-05,
-      "learning_rate": 7.985503068482974e-07,
-      "loss": 0.014609627425670624,
-      "num_tokens": 195544.0,
-      "reward": 0.5289265513420105,
-      "reward_std": 0.3883950710296631,
-      "rewards/true_env_reward_fn/mean": 0.5289265513420105,
-      "rewards/true_env_reward_fn/std": 0.3883951008319855,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 76.1875,
+      "completions/mean_terminated_length": 76.1875,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3037313222885132,
+      "epoch": 1.146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.061912886798381805,
+      "kl": 1.283655774386716e-05,
+      "learning_rate": 9.995470383368808e-07,
+      "loss": -0.01992109790444374,
+      "num_tokens": 1162249.0,
+      "reward": 0.49922606348991394,
+      "reward_std": 0.2621309757232666,
+      "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+      "rewards/true_env_reward_fn/std": 0.2621309757232666,
       "step": 47,
-      "step_time": 2.938600743000279
+      "step_time": 12.964419044000124
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 67.375,
-      "completions/mean_terminated_length": 67.375,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.5243317484855652,
-      "epoch": 0.3902439024390244,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20161111652851105,
-      "kl": 1.4497059055429418e-05,
-      "learning_rate": 7.982105091234235e-07,
-      "loss": 0.23342597484588623,
-      "num_tokens": 198691.0,
-      "reward": 0.45001715421676636,
-      "reward_std": 0.2565726041793823,
-      "rewards/true_env_reward_fn/mean": 0.45001715421676636,
-      "rewards/true_env_reward_fn/std": 0.2565726041793823,
+      "completions/max_length": 231.0,
+      "completions/max_terminated_length": 231.0,
+      "completions/mean_length": 71.375,
+      "completions/mean_terminated_length": 71.375,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2007178366184235,
+      "epoch": 1.170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0889662653207779,
+      "kl": 1.6228528693318367e-05,
+      "learning_rate": 9.993477785446149e-07,
+      "loss": 0.045945264399051666,
+      "num_tokens": 1184555.0,
+      "reward": 0.42501482367515564,
+      "reward_std": 0.27350595593452454,
+      "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+      "rewards/true_env_reward_fn/std": 0.27350592613220215,
       "step": 48,
-      "step_time": 4.91795033499875
+      "step_time": 17.23041258299986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 55.375,
-      "completions/mean_terminated_length": 55.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2544435858726501,
-      "epoch": 0.3983739837398374,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11793916672468185,
-      "kl": 1.3676196886081016e-05,
-      "learning_rate": 7.978350553629554e-07,
-      "loss": -0.016418367624282837,
-      "num_tokens": 202994.0,
-      "reward": 0.4054500162601471,
-      "reward_std": 0.20634961128234863,
-      "rewards/true_env_reward_fn/mean": 0.4054500162601471,
-      "rewards/true_env_reward_fn/std": 0.20634961128234863,
+      "completions/max_length": 86.0,
+      "completions/max_terminated_length": 86.0,
+      "completions/mean_length": 55.9375,
+      "completions/mean_terminated_length": 55.9375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.182040810585022,
+      "epoch": 1.1951219512195121,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08547856658697128,
+      "kl": 1.571832831359643e-05,
+      "learning_rate": 9.991123238414453e-07,
+      "loss": 0.02548346482217312,
+      "num_tokens": 1208384.0,
+      "reward": 0.3845663070678711,
+      "reward_std": 0.315467894077301,
+      "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+      "rewards/true_env_reward_fn/std": 0.31546786427497864,
       "step": 49,
-      "step_time": 3.626596234000317
+      "step_time": 8.691208415999881
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.022342562675476,
-      "epoch": 0.4065040650406504,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16596083343029022,
-      "kl": 1.1194244052603608e-05,
-      "learning_rate": 7.974239791852739e-07,
-      "loss": 0.0499756895005703,
-      "num_tokens": 205770.0,
-      "reward": 0.5639185309410095,
-      "reward_std": 0.1721728891134262,
-      "rewards/true_env_reward_fn/mean": 0.5639185309410095,
-      "rewards/true_env_reward_fn/std": 0.1721728891134262,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 64.75,
+      "completions/mean_terminated_length": 64.75,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2306177020072937,
+      "epoch": 1.2195121951219512,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07395736873149872,
+      "kl": 1.2643881973417592e-05,
+      "learning_rate": 9.988406912941589e-07,
+      "loss": -0.04186868295073509,
+      "num_tokens": 1227700.0,
+      "reward": 0.5068289637565613,
+      "reward_std": 0.31324177980422974,
+      "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+      "rewards/true_env_reward_fn/std": 0.31324175000190735,
       "step": 50,
-      "step_time": 3.9679293660010444
+      "step_time": 10.162109979000206
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 60.625,
-      "completions/mean_terminated_length": 60.625,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 0.9613964557647705,
-      "epoch": 0.4146341463414634,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12812700867652893,
-      "kl": 1.0464088063599775e-05,
-      "learning_rate": 7.969773173984153e-07,
-      "loss": 0.023206032812595367,
-      "num_tokens": 210443.0,
-      "reward": 0.3208000063896179,
-      "reward_std": 0.25050169229507446,
-      "rewards/true_env_reward_fn/mean": 0.3208000063896179,
-      "rewards/true_env_reward_fn/std": 0.25050172209739685,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 61.25,
+      "completions/mean_terminated_length": 61.25,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 1.2760189771652222,
+      "epoch": 1.2439024390243902,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0870513767004013,
+      "kl": 1.4371181578098913e-05,
+      "learning_rate": 9.985329005918702e-07,
+      "loss": -0.01623840071260929,
+      "num_tokens": 1253120.0,
+      "reward": 0.3888077139854431,
+      "reward_std": 0.3346175253391266,
+      "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+      "rewards/true_env_reward_fn/std": 0.3346175253391266,
       "step": 51,
-      "step_time": 3.6275602460009395
+      "step_time": 10.88732858200001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.2359730005264282,
-      "epoch": 0.42276422764227645,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1384950578212738,
-      "kl": 1.2094554222130682e-05,
-      "learning_rate": 7.964951099967749e-07,
-      "loss": -0.07054222375154495,
-      "num_tokens": 213833.0,
-      "reward": 0.5900156497955322,
-      "reward_std": 0.18237514793872833,
-      "rewards/true_env_reward_fn/mean": 0.5900156497955322,
-      "rewards/true_env_reward_fn/std": 0.18237514793872833,
+      "completions/max_length": 93.0,
+      "completions/max_terminated_length": 93.0,
+      "completions/mean_length": 52.72916793823242,
+      "completions/mean_terminated_length": 52.72916793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.1669773161411285,
+      "epoch": 1.2682926829268293,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.1055479422211647,
+      "kl": 1.69047059443983e-05,
+      "learning_rate": 9.981889740445957e-07,
+      "loss": 0.03519687056541443,
+      "num_tokens": 1274803.0,
+      "reward": 0.4995749592781067,
+      "reward_std": 0.2088174670934677,
+      "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+      "rewards/true_env_reward_fn/std": 0.2088174819946289,
       "step": 52,
-      "step_time": 3.8849526029989647
+      "step_time": 9.252596976000177
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 79.5,
-      "completions/mean_terminated_length": 79.5,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.2706108689308167,
-      "epoch": 0.43089430894308944,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17388403415679932,
-      "kl": 1.3583369309344562e-05,
-      "learning_rate": 7.959774001575264e-07,
-      "loss": 0.06114684417843819,
-      "num_tokens": 216853.0,
-      "reward": 0.4848448634147644,
-      "reward_std": 0.2859330177307129,
-      "rewards/true_env_reward_fn/mean": 0.4848448634147644,
-      "rewards/true_env_reward_fn/std": 0.2859330177307129,
+      "completions/max_length": 310.0,
+      "completions/max_terminated_length": 310.0,
+      "completions/mean_length": 76.72917175292969,
+      "completions/mean_terminated_length": 76.72917175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2451297044754028,
+      "epoch": 1.2926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07513763755559921,
+      "kl": 1.5911174841676257e-05,
+      "learning_rate": 9.978089365816355e-07,
+      "loss": 0.07006432861089706,
+      "num_tokens": 1305910.0,
+      "reward": 0.33895593881607056,
+      "reward_std": 0.2969740927219391,
+      "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+      "rewards/true_env_reward_fn/std": 0.2969740927219391,
       "step": 53,
-      "step_time": 4.964324356000361
+      "step_time": 24.22518693999973
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 64.875,
-      "completions/mean_terminated_length": 64.875,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2430712580680847,
-      "epoch": 0.43902439024390244,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11113106459379196,
-      "kl": 1.2204414360894589e-05,
-      "learning_rate": 7.954242342367553e-07,
-      "loss": 0.010590985417366028,
-      "num_tokens": 221252.0,
-      "reward": 0.392258882522583,
-      "reward_std": 0.13280020654201508,
-      "rewards/true_env_reward_fn/mean": 0.392258882522583,
-      "rewards/true_env_reward_fn/std": 0.13280019164085388,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 66.02083587646484,
+      "completions/mean_terminated_length": 66.02083587646484,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.3110129833221436,
+      "epoch": 1.3170731707317074,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0695035383105278,
+      "kl": 1.606306568646687e-05,
+      "learning_rate": 9.973928157497674e-07,
+      "loss": 0.03299739956855774,
+      "num_tokens": 1330815.0,
+      "reward": 0.4440445899963379,
+      "reward_std": 0.2889502942562103,
+      "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+      "rewards/true_env_reward_fn/std": 0.2889502942562103,
       "step": 54,
-      "step_time": 3.5511989209990134
+      "step_time": 10.14821418500037
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 118.0,
-      "completions/max_terminated_length": 118.0,
-      "completions/mean_length": 61.875,
-      "completions/mean_terminated_length": 61.875,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.3175880908966064,
-      "epoch": 0.44715447154471544,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20881597697734833,
-      "kl": 1.58558846123924e-05,
-      "learning_rate": 7.948356617653087e-07,
-      "loss": -0.06772151589393616,
-      "num_tokens": 224691.0,
-      "reward": 0.30961817502975464,
-      "reward_std": 0.27422165870666504,
-      "rewards/true_env_reward_fn/mean": 0.30961817502975464,
-      "rewards/true_env_reward_fn/std": 0.27422165870666504,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 61.958335876464844,
+      "completions/mean_terminated_length": 61.958335876464844,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2969173192977905,
+      "epoch": 1.3414634146341464,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09475436061620712,
+      "kl": 1.5850717545617954e-05,
+      "learning_rate": 9.969406417112488e-07,
+      "loss": -0.014009319245815277,
+      "num_tokens": 1361885.0,
+      "reward": 0.3289160430431366,
+      "reward_std": 0.26591774821281433,
+      "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+      "rewards/true_env_reward_fn/std": 0.2659177780151367,
       "step": 55,
-      "step_time": 5.031640098000935
+      "step_time": 11.13082981000025
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 64.625,
-      "completions/mean_terminated_length": 64.625,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.4056915640830994,
-      "epoch": 0.45528455284552843,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.606108895037323e-05,
-      "kl": 1.2847603557020193e-05,
-      "learning_rate": 7.942117354443597e-07,
-      "loss": 6.408997705875663e-07,
-      "num_tokens": 228116.0,
-      "reward": 0.5599601864814758,
-      "reward_std": 0.18355616927146912,
-      "rewards/true_env_reward_fn/mean": 0.5599601864814758,
-      "rewards/true_env_reward_fn/std": 0.18355616927146912,
+      "completions/max_length": 154.0,
+      "completions/max_terminated_length": 154.0,
+      "completions/mean_length": 80.22917175292969,
+      "completions/mean_terminated_length": 80.22917175292969,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.2630544006824493,
+      "epoch": 1.3658536585365852,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.06734384596347809,
+      "kl": 1.5091616887730197e-05,
+      "learning_rate": 9.964524472416317e-07,
+      "loss": -0.10958556830883026,
+      "num_tokens": 1390496.0,
+      "reward": 0.46485185623168945,
+      "reward_std": 0.29441413283348083,
+      "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+      "rewards/true_env_reward_fn/std": 0.29441413283348083,
       "step": 56,
-      "step_time": 3.6221305880008003
+      "step_time": 14.49393488900023
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 59.875,
-      "completions/mean_terminated_length": 59.875,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.4034882187843323,
-      "epoch": 0.4634146341463415,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19157616794109344,
-      "kl": 1.4551038475474343e-05,
-      "learning_rate": 7.935525111406885e-07,
-      "loss": 0.021202675998210907,
-      "num_tokens": 233139.0,
-      "reward": 0.32785865664482117,
-      "reward_std": 0.2835054397583008,
-      "rewards/true_env_reward_fn/mean": 0.32785865664482117,
-      "rewards/true_env_reward_fn/std": 0.28350546956062317,
+      "completions/max_length": 203.0,
+      "completions/max_terminated_length": 203.0,
+      "completions/mean_length": 64.27083587646484,
+      "completions/mean_terminated_length": 64.27083587646484,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.1640427708625793,
+      "epoch": 1.3902439024390243,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07240130007266998,
+      "kl": 1.509602225269191e-05,
+      "learning_rate": 9.959282677273868e-07,
+      "loss": 0.10520926117897034,
+      "num_tokens": 1411837.0,
+      "reward": 0.5296112895011902,
+      "reward_std": 0.2505757212638855,
+      "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+      "rewards/true_env_reward_fn/std": 0.2505757212638855,
       "step": 57,
-      "step_time": 3.7005361410010664
+      "step_time": 14.065935286000013
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 173.0,
-      "completions/max_terminated_length": 173.0,
-      "completions/mean_length": 76.625,
-      "completions/mean_terminated_length": 76.625,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2941595911979675,
-      "epoch": 0.4715447154471545,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 65.58333587646484,
+      "completions/mean_terminated_length": 65.58333587646484,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.1222519278526306,
+      "epoch": 1.4146341463414633,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14299193024635315,
-      "kl": 1.3164159554435173e-05,
-      "learning_rate": 7.92858047881681e-07,
-      "loss": -0.14726585149765015,
-      "num_tokens": 238584.0,
-      "reward": 0.444433331489563,
-      "reward_std": 0.030650291591882706,
-      "rewards/true_env_reward_fn/mean": 0.444433331489563,
-      "rewards/true_env_reward_fn/std": 0.030650287866592407,
+      "grad_norm": 0.05992415174841881,
+      "kl": 1.2099166724510724e-05,
+      "learning_rate": 9.953681411633374e-07,
+      "loss": 0.004622246604412794,
+      "num_tokens": 1438569.0,
+      "reward": 0.41778087615966797,
+      "reward_std": 0.28395572304725647,
+      "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+      "rewards/true_env_reward_fn/std": 0.2839556932449341,
       "step": 58,
-      "step_time": 7.550715425000817
+      "step_time": 12.76291504100027
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 54.625,
-      "completions/mean_terminated_length": 54.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1917714476585388,
-      "epoch": 0.4796747967479675,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.25083038210868835,
-      "kl": 1.3176229913369752e-05,
-      "learning_rate": 7.921284078500422e-07,
-      "loss": 0.088463693857193,
-      "num_tokens": 240669.0,
-      "reward": 0.7982887029647827,
-      "reward_std": 0.1672983169555664,
-      "rewards/true_env_reward_fn/mean": 0.7982887029647827,
-      "rewards/true_env_reward_fn/std": 0.1672983169555664,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 75.33333587646484,
+      "completions/mean_terminated_length": 75.33333587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.1821540892124176,
+      "epoch": 1.4390243902439024,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0654020607471466,
+      "kl": 1.4932766589481616e-05,
+      "learning_rate": 9.947721081499067e-07,
+      "loss": 0.06719422340393066,
+      "num_tokens": 1461033.0,
+      "reward": 0.5268458127975464,
+      "reward_std": 0.23783695697784424,
+      "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+      "rewards/true_env_reward_fn/std": 0.23783694207668304,
       "step": 59,
-      "step_time": 3.7769912429994292
+      "step_time": 11.089177285999767
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 66.375,
-      "completions/mean_terminated_length": 66.375,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.3743653893470764,
-      "epoch": 0.4878048780487805,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18919643759727478,
-      "kl": 1.231462101713987e-05,
-      "learning_rate": 7.91363656378229e-07,
-      "loss": -0.08548973500728607,
-      "num_tokens": 243808.0,
-      "reward": 0.5988538861274719,
-      "reward_std": 0.11870570480823517,
-      "rewards/true_env_reward_fn/mean": 0.5988538861274719,
-      "rewards/true_env_reward_fn/std": 0.11870571970939636,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 72.14583587646484,
+      "completions/mean_terminated_length": 72.14583587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2446446418762207,
+      "epoch": 1.4634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08000538498163223,
+      "kl": 1.3416995898296591e-05,
+      "learning_rate": 9.941402118901742e-07,
+      "loss": 0.05287330970168114,
+      "num_tokens": 1488264.0,
+      "reward": 0.4032561779022217,
+      "reward_std": 0.24067741632461548,
+      "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+      "rewards/true_env_reward_fn/std": 0.24067740142345428,
       "step": 60,
-      "step_time": 4.052767743998629
+      "step_time": 12.328215124000053
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 132.0,
-      "completions/max_terminated_length": 132.0,
-      "completions/mean_length": 77.125,
-      "completions/mean_terminated_length": 77.125,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.329764723777771,
-      "epoch": 0.4959349593495935,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1690233051776886,
-      "kl": 1.405783814334427e-05,
-      "learning_rate": 7.905638619426003e-07,
-      "loss": 0.0050433604046702385,
-      "num_tokens": 248725.0,
-      "reward": 0.27516257762908936,
-      "reward_std": 0.32322537899017334,
-      "rewards/true_env_reward_fn/mean": 0.27516257762908936,
-      "rewards/true_env_reward_fn/std": 0.32322537899017334,
+      "completions/max_length": 191.0,
+      "completions/max_terminated_length": 191.0,
+      "completions/mean_length": 65.8125,
+      "completions/mean_terminated_length": 65.8125,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.416578859090805,
+      "epoch": 1.4878048780487805,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07916785031557083,
+      "kl": 1.8312134670850355e-05,
+      "learning_rate": 9.934724981867446e-07,
+      "loss": -0.02956264466047287,
+      "num_tokens": 1506607.0,
+      "reward": 0.6846215724945068,
+      "reward_std": 0.21603551506996155,
+      "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+      "rewards/true_env_reward_fn/std": 0.21603552997112274,
       "step": 61,
-      "step_time": 6.010593229999358
+      "step_time": 13.294195681999781
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.2542970776557922,
-      "epoch": 0.5040650406504065,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11462891101837158,
-      "kl": 1.13775058707688e-05,
-      "learning_rate": 7.897290961572853e-07,
-      "loss": -0.007184989750385284,
-      "num_tokens": 252101.0,
-      "reward": 0.5372380018234253,
-      "reward_std": 0.13500821590423584,
-      "rewards/true_env_reward_fn/mean": 0.5372380018234253,
-      "rewards/true_env_reward_fn/std": 0.13500821590423584,
+      "completions/max_length": 182.0,
+      "completions/max_terminated_length": 182.0,
+      "completions/mean_length": 64.33333587646484,
+      "completions/mean_terminated_length": 64.33333587646484,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2478635609149933,
+      "epoch": 1.5121951219512195,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09035050123929977,
+      "kl": 1.867344440142915e-05,
+      "learning_rate": 9.927690154384272e-07,
+      "loss": -0.048415351659059525,
+      "num_tokens": 1539351.0,
+      "reward": 0.17628252506256104,
+      "reward_std": 0.2993278205394745,
+      "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+      "rewards/true_env_reward_fn/std": 0.2993278503417969,
       "step": 62,
-      "step_time": 3.4512634010006877
+      "step_time": 18.972790307999958
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 65.75,
-      "completions/mean_terminated_length": 65.75,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.1982964873313904,
-      "epoch": 0.5121951219512195,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12309258431196213,
-      "kl": 1.69004347299051e-05,
-      "learning_rate": 7.888594337677712e-07,
-      "loss": 0.0009508281946182251,
-      "num_tokens": 255231.0,
-      "reward": 0.6114543080329895,
-      "reward_std": 0.10413603484630585,
-      "rewards/true_env_reward_fn/mean": 0.6114543080329895,
-      "rewards/true_env_reward_fn/std": 0.10413603484630585,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 61.66666793823242,
+      "completions/mean_terminated_length": 61.66666793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2024545669555664,
+      "epoch": 1.5365853658536586,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09254598617553711,
+      "kl": 1.654068455536617e-05,
+      "learning_rate": 9.920298146367286e-07,
+      "loss": 0.09414710104465485,
+      "num_tokens": 1565215.0,
+      "reward": 0.4147046208381653,
+      "reward_std": 0.1770697683095932,
+      "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+      "rewards/true_env_reward_fn/std": 0.1770697683095932,
       "step": 63,
-      "step_time": 3.735559521997857
+      "step_time": 11.292631820999986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 129.0,
-      "completions/max_terminated_length": 129.0,
-      "completions/mean_length": 69.0,
-      "completions/mean_terminated_length": 69.0,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.3587612509727478,
-      "epoch": 0.5203252032520326,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15858450531959534,
-      "kl": 1.4598341294913553e-05,
-      "learning_rate": 7.879549526442108e-07,
-      "loss": 0.0696716383099556,
-      "num_tokens": 260523.0,
-      "reward": 0.2912999987602234,
-      "reward_std": 0.2844822406768799,
-      "rewards/true_env_reward_fn/mean": 0.2912999987602234,
-      "rewards/true_env_reward_fn/std": 0.2844822406768799,
+      "completions/max_length": 120.0,
+      "completions/max_terminated_length": 120.0,
+      "completions/mean_length": 64.1875,
+      "completions/mean_terminated_length": 64.1875,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2442612051963806,
+      "epoch": 1.5609756097560976,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.052295491099357605,
+      "kl": 1.716045289867907e-05,
+      "learning_rate": 9.912549493621554e-07,
+      "loss": 0.01475335843861103,
+      "num_tokens": 1589608.0,
+      "reward": 0.48741206526756287,
+      "reward_std": 0.21404753625392914,
+      "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+      "rewards/true_env_reward_fn/std": 0.21404753625392914,
       "step": 64,
-      "step_time": 5.731267729999672
+      "step_time": 9.600786530999585
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 230.0,
-      "completions/max_terminated_length": 230.0,
-      "completions/mean_length": 83.5,
-      "completions/mean_terminated_length": 83.5,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2284430861473083,
-      "epoch": 0.5284552845528455,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13374178111553192,
-      "kl": 1.2341822639427846e-05,
-      "learning_rate": 7.870157337744494e-07,
-      "loss": 0.10693901032209396,
-      "num_tokens": 264967.0,
-      "reward": 0.3284733295440674,
-      "reward_std": 0.3848404288291931,
-      "rewards/true_env_reward_fn/mean": 0.3284733295440674,
-      "rewards/true_env_reward_fn/std": 0.3848404288291931,
+      "completions/max_length": 111.0,
+      "completions/max_terminated_length": 111.0,
+      "completions/mean_length": 61.3125,
+      "completions/mean_terminated_length": 61.3125,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.2735203504562378,
+      "epoch": 1.5853658536585367,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.094636932015419,
+      "kl": 2.017962560785236e-05,
+      "learning_rate": 9.90444475780332e-07,
+      "loss": -0.0016674790531396866,
+      "num_tokens": 1619095.0,
+      "reward": 0.34675830602645874,
+      "reward_std": 0.2556215822696686,
+      "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+      "rewards/true_env_reward_fn/std": 0.2556215822696686,
       "step": 65,
-      "step_time": 9.601442954000959
+      "step_time": 10.169144185999812
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 132.0,
-      "completions/max_terminated_length": 132.0,
-      "completions/mean_length": 85.25,
-      "completions/mean_terminated_length": 85.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2396279573440552,
-      "epoch": 0.5365853658536586,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.08427461981773376,
-      "kl": 1.4658115105703473e-05,
-      "learning_rate": 7.860418612567733e-07,
-      "loss": -0.05642998591065407,
-      "num_tokens": 269717.0,
-      "reward": 0.38946664333343506,
-      "reward_std": 0.1897086799144745,
-      "rewards/true_env_reward_fn/mean": 0.38946664333343506,
-      "rewards/true_env_reward_fn/std": 0.1897086799144745,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 65.9375,
+      "completions/mean_terminated_length": 65.9375,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2845994234085083,
+      "epoch": 1.6097560975609757,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05164559930562973,
+      "kl": 1.66792838172114e-05,
+      "learning_rate": 9.89598452637928e-07,
+      "loss": 0.011961851269006729,
+      "num_tokens": 1645076.0,
+      "reward": 0.44275379180908203,
+      "reward_std": 0.2063576877117157,
+      "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+      "rewards/true_env_reward_fn/std": 0.2063576877117157,
       "step": 66,
-      "step_time": 6.017849919000582
+      "step_time": 10.343706631000032
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 72.25,
-      "completions/mean_terminated_length": 72.25,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.2245049476623535,
-      "epoch": 0.5447154471544715,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13357418775558472,
-      "kl": 1.2806529412046075e-05,
-      "learning_rate": 7.850334222923798e-07,
-      "loss": 0.03744228184223175,
-      "num_tokens": 275407.0,
-      "reward": 0.08966667205095291,
-      "reward_std": 0.23612774908542633,
-      "rewards/true_env_reward_fn/mean": 0.08966667205095291,
-      "rewards/true_env_reward_fn/std": 0.23612776398658752,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 66.0625,
+      "completions/mean_terminated_length": 66.0625,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3418844938278198,
+      "epoch": 1.6341463414634148,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05729615315794945,
+      "kl": 1.6437259546364658e-05,
+      "learning_rate": 9.88716941258401e-07,
+      "loss": 0.015346314758062363,
+      "num_tokens": 1675423.0,
+      "reward": 0.4190921187400818,
+      "reward_std": 0.3388116955757141,
+      "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+      "rewards/true_env_reward_fn/std": 0.3388116657733917,
       "step": 67,
-      "step_time": 4.4363536659984675
+      "step_time": 12.16719347000003
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 56.875,
-      "completions/mean_terminated_length": 56.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.2679882645606995,
-      "epoch": 0.5528455284552846,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15667268633842468,
-      "kl": 1.2213955869810889e-05,
-      "learning_rate": 7.83990507177569e-07,
-      "loss": -0.052396662533283234,
-      "num_tokens": 280838.0,
-      "reward": 0.2431039959192276,
-      "reward_std": 0.2672288715839386,
-      "rewards/true_env_reward_fn/mean": 0.2431039959192276,
-      "rewards/true_env_reward_fn/std": 0.2672288715839386,
+      "completions/max_length": 230.0,
+      "completions/max_terminated_length": 230.0,
+      "completions/mean_length": 78.66667175292969,
+      "completions/mean_terminated_length": 78.66667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2716343402862549,
+      "epoch": 1.6585365853658538,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07959166914224625,
+      "kl": 1.785568679224525e-05,
+      "learning_rate": 9.87800005537551e-07,
+      "loss": 0.08119910955429077,
+      "num_tokens": 1698103.0,
+      "reward": 0.5178458094596863,
+      "reward_std": 0.22635267674922943,
+      "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+      "rewards/true_env_reward_fn/std": 0.22635267674922943,
       "step": 68,
-      "step_time": 3.6370441849994677
+      "step_time": 15.523659553000016
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 62.625,
-      "completions/mean_terminated_length": 62.625,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.2563416361808777,
-      "epoch": 0.5609756097560976,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.937557868193835e-05,
-      "kl": 1.1138304216729011e-05,
-      "learning_rate": 7.829132092956586e-07,
-      "loss": 5.569941095018294e-07,
-      "num_tokens": 283603.0,
-      "reward": 0.6040733456611633,
-      "reward_std": 0.0834638923406601,
-      "rewards/true_env_reward_fn/mean": 0.6040733456611633,
-      "rewards/true_env_reward_fn/std": 0.08346389979124069,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 61.583335876464844,
+      "completions/mean_terminated_length": 61.583335876464844,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 1.1755708158016205,
+      "epoch": 1.6829268292682928,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08657841384410858,
+      "kl": 2.0373249526528525e-05,
+      "learning_rate": 9.868477119388894e-07,
+      "loss": -0.01668858528137207,
+      "num_tokens": 1723155.0,
+      "reward": 0.39579567313194275,
+      "reward_std": 0.3625684082508087,
+      "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+      "rewards/true_env_reward_fn/std": 0.3625684380531311,
       "step": 69,
-      "step_time": 3.466609713001162
+      "step_time": 9.677535032999685
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 189.0,
-      "completions/max_terminated_length": 189.0,
-      "completions/mean_length": 126.0,
-      "completions/mean_terminated_length": 126.0,
-      "completions/min_length": 76.0,
-      "completions/min_terminated_length": 76.0,
-      "entropy": 1.8668264746665955,
-      "epoch": 0.5691056910569106,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11100972443819046,
-      "kl": 1.3833193406753708e-05,
-      "learning_rate": 7.81801625108622e-07,
-      "loss": -0.04258224368095398,
-      "num_tokens": 290511.0,
-      "reward": 0.37345871329307556,
-      "reward_std": 0.016035744920372963,
-      "rewards/true_env_reward_fn/mean": 0.37345871329307556,
-      "rewards/true_env_reward_fn/std": 0.016035741195082664,
+      "completions/max_length": 212.0,
+      "completions/max_terminated_length": 212.0,
+      "completions/mean_length": 74.79167175292969,
+      "completions/mean_terminated_length": 74.79167175292969,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.296934336423874,
+      "epoch": 1.7073170731707317,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07736476510763168,
+      "kl": 2.0918997506669257e-05,
+      "learning_rate": 9.85860129488821e-07,
+      "loss": 0.03239107131958008,
+      "num_tokens": 1742689.0,
+      "reward": 0.6141302585601807,
+      "reward_std": 0.23138943314552307,
+      "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+      "rewards/true_env_reward_fn/std": 0.23138941824436188,
       "step": 70,
-      "step_time": 8.357124549000218
+      "step_time": 15.20990351499995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 58.5,
-      "completions/mean_terminated_length": 58.5,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.1647167801856995,
-      "epoch": 0.5772357723577236,
+      "completions/max_length": 210.0,
+      "completions/max_terminated_length": 210.0,
+      "completions/mean_length": 66.125,
+      "completions/mean_terminated_length": 66.125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2145576775074005,
+      "epoch": 1.7317073170731707,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12842044234275818,
-      "kl": 1.35402724481537e-05,
-      "learning_rate": 7.806558541484517e-07,
-      "loss": -0.0010651163756847382,
-      "num_tokens": 294315.0,
-      "reward": 0.6432806849479675,
-      "reward_std": 0.2300010770559311,
-      "rewards/true_env_reward_fn/mean": 0.6432806849479675,
-      "rewards/true_env_reward_fn/std": 0.23000109195709229,
+      "grad_norm": 0.07104668766260147,
+      "kl": 1.726100731502811e-05,
+      "learning_rate": 9.848373297716414e-07,
+      "loss": 0.03256790712475777,
+      "num_tokens": 1765463.0,
+      "reward": 0.48419874906539917,
+      "reward_std": 0.32040080428123474,
+      "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+      "rewards/true_env_reward_fn/std": 0.32040080428123474,
       "step": 71,
-      "step_time": 3.8402047919989855
+      "step_time": 14.703903473999844
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.1465299725532532,
-      "epoch": 0.5853658536585366,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23560228943824768,
-      "kl": 1.4576367902918719e-05,
-      "learning_rate": 7.794759990082466e-07,
-      "loss": -0.11232151836156845,
-      "num_tokens": 297803.0,
-      "reward": 0.30700522661209106,
-      "reward_std": 0.3690750300884247,
-      "rewards/true_env_reward_fn/mean": 0.30700522661209106,
-      "rewards/true_env_reward_fn/std": 0.3690750002861023,
+      "completions/max_length": 86.0,
+      "completions/max_terminated_length": 86.0,
+      "completions/mean_length": 62.35416793823242,
+      "completions/mean_terminated_length": 62.35416793823242,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.265857070684433,
+      "epoch": 1.7560975609756098,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07747533172369003,
+      "kl": 1.5618132920280914e-05,
+      "learning_rate": 9.837793869243467e-07,
+      "loss": -0.00018438976258039474,
+      "num_tokens": 1791512.0,
+      "reward": 0.45079630613327026,
+      "reward_std": 0.2226068526506424,
+      "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+      "rewards/true_env_reward_fn/std": 0.2226068526506424,
       "step": 72,
-      "step_time": 3.467162693001228
+      "step_time": 9.422353613000269
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 137.0,
-      "completions/max_terminated_length": 137.0,
-      "completions/mean_length": 73.5,
-      "completions/mean_terminated_length": 73.5,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "entropy": 1.2479569911956787,
-      "epoch": 0.5934959349593496,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011426099081290886,
-      "kl": 1.304310217165039e-05,
-      "learning_rate": 7.782621653330256e-07,
-      "loss": 6.391838383024151e-07,
-      "num_tokens": 301427.0,
-      "reward": 0.5610077381134033,
-      "reward_std": 0.316459059715271,
-      "rewards/true_env_reward_fn/mean": 0.5610077381134033,
-      "rewards/true_env_reward_fn/std": 0.3164590299129486,
+      "completions/max_length": 188.0,
+      "completions/max_terminated_length": 188.0,
+      "completions/mean_length": 73.0625,
+      "completions/mean_terminated_length": 73.0625,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.3557232320308685,
+      "epoch": 1.7804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0811411589384079,
+      "kl": 1.9600361611082917e-05,
+      "learning_rate": 9.826863776312618e-07,
+      "loss": -0.019779374822974205,
+      "num_tokens": 1820731.0,
+      "reward": 0.431186318397522,
+      "reward_std": 0.23306044936180115,
+      "rewards/true_env_reward_fn/mean": 0.431186318397522,
+      "rewards/true_env_reward_fn/std": 0.23306044936180115,
       "step": 73,
-      "step_time": 5.824168748999
+      "step_time": 16.65922043799992
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 63.125,
-      "completions/mean_terminated_length": 63.125,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.170280933380127,
-      "epoch": 0.6016260162601627,
+      "completions/max_length": 215.0,
+      "completions/max_terminated_length": 215.0,
+      "completions/mean_length": 78.375,
+      "completions/mean_terminated_length": 78.375,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2767037451267242,
+      "epoch": 1.8048780487804879,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22593456506729126,
-      "kl": 2.0052431864314713e-05,
-      "learning_rate": 7.77014461810269e-07,
-      "loss": 0.16111303865909576,
-      "num_tokens": 305492.0,
-      "reward": 0.3909183144569397,
-      "reward_std": 0.21756574511528015,
-      "rewards/true_env_reward_fn/mean": 0.3909183144569397,
-      "rewards/true_env_reward_fn/std": 0.21756574511528015,
+      "grad_norm": 0.08133924007415771,
+      "kl": 1.8058163732348476e-05,
+      "learning_rate": 9.815583811184808e-07,
+      "loss": -0.02447839081287384,
+      "num_tokens": 1841389.0,
+      "reward": 0.5825158953666687,
+      "reward_std": 0.2041907161474228,
+      "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+      "rewards/true_env_reward_fn/std": 0.20419073104858398,
       "step": 74,
-      "step_time": 4.510902927002462
+      "step_time": 13.422026366999944
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2373355031013489,
-      "epoch": 0.6097560975609756,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.604005466215312e-05,
-      "kl": 1.0138399375136942e-05,
-      "learning_rate": 7.757330001601855e-07,
-      "loss": 5.069200028628984e-07,
-      "num_tokens": 309826.0,
-      "reward": 0.5905972719192505,
-      "reward_std": 0.15080371499061584,
-      "rewards/true_env_reward_fn/mean": 0.5905972719192505,
-      "rewards/true_env_reward_fn/std": 0.15080371499061584,
+      "completions/max_length": 90.0,
+      "completions/max_terminated_length": 90.0,
+      "completions/mean_length": 54.25,
+      "completions/mean_terminated_length": 54.25,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.2325710952281952,
+      "epoch": 1.8292682926829267,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.11014537513256073,
+      "kl": 3.268667387601454e-05,
+      "learning_rate": 9.803954791481238e-07,
+      "loss": 0.045359574258327484,
+      "num_tokens": 1871129.0,
+      "reward": 0.3935621678829193,
+      "reward_std": 0.22456605732440948,
+      "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+      "rewards/true_env_reward_fn/std": 0.2245660424232483,
       "step": 75,
-      "step_time": 3.6695911980004894
+      "step_time": 9.51117546200021
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.1873346865177155,
-      "epoch": 0.6178861788617886,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2067025899887085,
-      "kl": 1.4842833934380906e-05,
-      "learning_rate": 7.744178951257091e-07,
-      "loss": -0.036428727209568024,
-      "num_tokens": 316885.0,
-      "reward": 0.13499999046325684,
-      "reward_std": 0.23260429501533508,
-      "rewards/true_env_reward_fn/mean": 0.13499999046325684,
-      "rewards/true_env_reward_fn/std": 0.23260430991649628,
+      "completions/max_length": 134.0,
+      "completions/max_terminated_length": 134.0,
+      "completions/mean_length": 74.85417175292969,
+      "completions/mean_terminated_length": 74.85417175292969,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "entropy": 1.412343978881836,
+      "epoch": 1.8536585365853657,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07616850733757019,
+      "kl": 1.848336046350596e-05,
+      "learning_rate": 9.791977560124118e-07,
+      "loss": 0.030123719945549965,
+      "num_tokens": 1892706.0,
+      "reward": 0.5764689445495605,
+      "reward_std": 0.18864154815673828,
+      "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+      "rewards/true_env_reward_fn/std": 0.18864154815673828,
       "step": 76,
-      "step_time": 4.359561059001862
+      "step_time": 9.295928349999713
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0709484219551086,
-      "epoch": 0.6260162601626016,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18151503801345825,
-      "kl": 1.3910183042753488e-05,
-      "learning_rate": 7.730692644622251e-07,
-      "loss": -0.06179043650627136,
-      "num_tokens": 319230.0,
-      "reward": 0.6732838153839111,
-      "reward_std": 0.1450435221195221,
-      "rewards/true_env_reward_fn/mean": 0.6732838153839111,
-      "rewards/true_env_reward_fn/std": 0.14504355192184448,
+      "completions/max_length": 248.0,
+      "completions/max_terminated_length": 248.0,
+      "completions/mean_length": 71.54167175292969,
+      "completions/mean_terminated_length": 71.54167175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.342492938041687,
+      "epoch": 1.8780487804878048,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08141017705202103,
+      "kl": 1.587149881743244e-05,
+      "learning_rate": 9.779652985275562e-07,
+      "loss": -0.02852344512939453,
+      "num_tokens": 1915324.0,
+      "reward": 0.4926157593727112,
+      "reward_std": 0.20701222121715546,
+      "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+      "rewards/true_env_reward_fn/std": 0.20701222121715546,
       "step": 77,
-      "step_time": 3.1786108079995756
+      "step_time": 15.693113021000045
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 107.0,
-      "completions/max_terminated_length": 107.0,
-      "completions/mean_length": 72.375,
-      "completions/mean_terminated_length": 72.375,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.5439093112945557,
-      "epoch": 0.6341463414634146,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20670665800571442,
-      "kl": 1.7317805031780154e-05,
-      "learning_rate": 7.716872289270261e-07,
-      "loss": -0.0654018223285675,
-      "num_tokens": 324633.0,
-      "reward": 0.23838475346565247,
-      "reward_std": 0.2594907879829407,
-      "rewards/true_env_reward_fn/mean": 0.23838475346565247,
-      "rewards/true_env_reward_fn/std": 0.2594907879829407,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 256.0,
+      "completions/mean_length": 62.3125,
+      "completions/mean_terminated_length": 62.3125,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2102001011371613,
+      "epoch": 1.9024390243902438,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0555732287466526,
+      "kl": 1.6820984001242323e-05,
+      "learning_rate": 9.766981960274652e-07,
+      "loss": 0.041817761957645416,
+      "num_tokens": 1933867.0,
+      "reward": 0.5576165318489075,
+      "reward_std": 0.3197881579399109,
+      "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+      "rewards/true_env_reward_fn/std": 0.3197881281375885,
       "step": 78,
-      "step_time": 4.930556027000421
+      "step_time": 16.146651725000083
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 47.625,
-      "completions/mean_terminated_length": 47.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1996066868305206,
-      "epoch": 0.6422764227642277,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21137002110481262,
-      "kl": 1.325221819570288e-05,
-      "learning_rate": 7.702719122684991e-07,
-      "loss": 0.003889208659529686,
-      "num_tokens": 329142.0,
-      "reward": 0.3934500217437744,
-      "reward_std": 0.1389254629611969,
-      "rewards/true_env_reward_fn/mean": 0.3934500217437744,
-      "rewards/true_env_reward_fn/std": 0.1389254778623581,
+      "completions/max_length": 148.0,
+      "completions/max_terminated_length": 148.0,
+      "completions/mean_length": 69.25,
+      "completions/mean_terminated_length": 69.25,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2909597754478455,
+      "epoch": 1.9268292682926829,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05769016966223717,
+      "kl": 1.534885905130068e-05,
+      "learning_rate": 9.753965403572702e-07,
+      "loss": -0.04179058223962784,
+      "num_tokens": 1953375.0,
+      "reward": 0.5617212057113647,
+      "reward_std": 0.18222570419311523,
+      "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+      "rewards/true_env_reward_fn/std": 0.18222568929195404,
       "step": 79,
-      "step_time": 3.5688320999997813
+      "step_time": 9.82867347299998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.125,
-      "completions/mean_terminated_length": 53.125,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.4094278812408447,
-      "epoch": 0.6504065040650406,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.17559278011322021,
-      "kl": 1.6261046312138205e-05,
-      "learning_rate": 7.688234412150453e-07,
-      "loss": -0.04887707903981209,
-      "num_tokens": 331663.0,
-      "reward": 0.49859046936035156,
-      "reward_std": 0.12171231955289841,
-      "rewards/true_env_reward_fn/mean": 0.49859046936035156,
-      "rewards/true_env_reward_fn/std": 0.12171231955289841,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 66.5,
+      "completions/mean_terminated_length": 66.5,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.2647078335285187,
+      "epoch": 1.951219512195122,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06353812664747238,
+      "kl": 2.195177648900426e-05,
+      "learning_rate": 9.740604258666668e-07,
+      "loss": -0.09541463106870651,
+      "num_tokens": 1978255.0,
+      "reward": 0.5184200406074524,
+      "reward_std": 0.28920501470565796,
+      "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+      "rewards/true_env_reward_fn/std": 0.28920501470565796,
       "step": 80,
-      "step_time": 3.7867210379990865
+      "step_time": 11.267316974000096
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 51.25,
-      "completions/mean_terminated_length": 51.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.1693094372749329,
-      "epoch": 0.6585365853658537,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010281114373356104,
-      "kl": 1.2930718639836414e-05,
-      "learning_rate": 7.673419454637328e-07,
-      "loss": 6.465359092544531e-07,
-      "num_tokens": 334637.0,
-      "reward": 0.5707399845123291,
-      "reward_std": 0.11909874528646469,
-      "rewards/true_env_reward_fn/mean": 0.5707399845123291,
-      "rewards/true_env_reward_fn/std": 0.1190987378358841,
+      "completions/max_length": 99.0,
+      "completions/max_terminated_length": 99.0,
+      "completions/mean_length": 62.72916793823242,
+      "completions/mean_terminated_length": 62.72916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3497782051563263,
+      "epoch": 1.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08975031226873398,
+      "kl": 3.0107988550298614e-05,
+      "learning_rate": 9.726899494030766e-07,
+      "loss": 0.04644065350294113,
+      "num_tokens": 2007634.0,
+      "reward": 0.3841831684112549,
+      "reward_std": 0.30559155344963074,
+      "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+      "rewards/true_env_reward_fn/std": 0.30559155344963074,
       "step": 81,
-      "step_time": 3.4751437539998733
+      "step_time": 10.035370067999793
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.3214005827903748,
-      "epoch": 0.6666666666666666,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 64.91667175292969,
+      "completions/mean_terminated_length": 64.91667175292969,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2758312821388245,
+      "epoch": 2.0,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2361973226070404,
-      "kl": 1.4227861356630456e-05,
-      "learning_rate": 7.658275576686829e-07,
-      "loss": -0.08402466773986816,
-      "num_tokens": 341701.0,
-      "reward": 0.09331665933132172,
-      "reward_std": 0.2172754853963852,
-      "rewards/true_env_reward_fn/mean": 0.09331665933132172,
-      "rewards/true_env_reward_fn/std": 0.2172755002975464,
+      "grad_norm": 0.08747493475675583,
+      "kl": 1.806905720513896e-05,
+      "learning_rate": 9.71285210304628e-07,
+      "loss": -0.07698298245668411,
+      "num_tokens": 2024382.0,
+      "reward": 0.6160596609115601,
+      "reward_std": 0.23944181203842163,
+      "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+      "rewards/true_env_reward_fn/std": 0.23944182693958282,
       "step": 82,
-      "step_time": 4.433740980000948
+      "step_time": 9.56242024800008
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 512.0,
-      "completions/max_terminated_length": 126.0,
-      "completions/mean_length": 142.375,
-      "completions/mean_terminated_length": 89.5714340209961,
-      "completions/min_length": 62.0,
-      "completions/min_terminated_length": 62.0,
-      "entropy": 1.817092776298523,
-      "epoch": 0.6747967479674797,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.11815346032381058,
-      "kl": 1.6899173715501092e-05,
-      "learning_rate": 7.642804134291927e-07,
-      "loss": -0.09939523041248322,
-      "num_tokens": 346380.0,
-      "reward": 0.47429025173187256,
-      "reward_std": 0.24831563234329224,
-      "rewards/true_env_reward_fn/mean": 0.47429025173187256,
-      "rewards/true_env_reward_fn/std": 0.24831561744213104,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 63.0625,
+      "completions/mean_terminated_length": 63.0625,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2893573343753815,
+      "epoch": 2.024390243902439,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05648891627788544,
+      "kl": 1.960936606337782e-05,
+      "learning_rate": 9.698463103929541e-07,
+      "loss": 0.05065512657165527,
+      "num_tokens": 2046817.0,
+      "reward": 0.5863184332847595,
+      "reward_std": 0.19063502550125122,
+      "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+      "rewards/true_env_reward_fn/std": 0.19063502550125122,
       "step": 83,
-      "step_time": 20.738665008999305
+      "step_time": 10.563381390999666
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 58.25,
-      "completions/mean_terminated_length": 58.25,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2211430668830872,
-      "epoch": 0.6829268292682927,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20952872931957245,
-      "kl": 1.2894654446427012e-05,
-      "learning_rate": 7.62700651277593e-07,
-      "loss": -0.0016747117042541504,
-      "num_tokens": 351186.0,
-      "reward": 0.386501669883728,
-      "reward_std": 0.17392057180404663,
-      "rewards/true_env_reward_fn/mean": 0.386501669883728,
-      "rewards/true_env_reward_fn/std": 0.17392057180404663,
+      "completions/max_length": 190.0,
+      "completions/max_terminated_length": 190.0,
+      "completions/mean_length": 75.29167175292969,
+      "completions/mean_terminated_length": 75.29167175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2169642746448517,
+      "epoch": 2.048780487804878,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06511837989091873,
+      "kl": 2.2800771603215253e-05,
+      "learning_rate": 9.683733539658138e-07,
+      "loss": 0.02157626487314701,
+      "num_tokens": 2074535.0,
+      "reward": 0.4389227330684662,
+      "reward_std": 0.303769588470459,
+      "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+      "rewards/true_env_reward_fn/std": 0.303769588470459,
       "step": 84,
-      "step_time": 4.028964023000299
+      "step_time": 17.21621736799966
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 66.625,
-      "completions/mean_terminated_length": 66.625,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.4367225170135498,
-      "epoch": 0.6910569105691057,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18939745426177979,
-      "kl": 1.6035403859859798e-05,
-      "learning_rate": 7.610884126668449e-07,
-      "loss": 0.0628451332449913,
-      "num_tokens": 355999.0,
-      "reward": 0.5092726349830627,
-      "reward_std": 0.2734805643558502,
-      "rewards/true_env_reward_fn/mean": 0.5092726349830627,
-      "rewards/true_env_reward_fn/std": 0.2734805941581726,
+      "completions/max_length": 432.0,
+      "completions/max_terminated_length": 432.0,
+      "completions/mean_length": 78.83333587646484,
+      "completions/mean_terminated_length": 78.83333587646484,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.396474927663803,
+      "epoch": 2.073170731707317,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.06777605414390564,
+      "kl": 2.369298363191774e-05,
+      "learning_rate": 9.66866447789531e-07,
+      "loss": -0.024554546922445297,
+      "num_tokens": 2096031.0,
+      "reward": 0.5134819746017456,
+      "reward_std": 0.28690314292907715,
+      "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+      "rewards/true_env_reward_fn/std": 0.28690314292907715,
       "step": 85,
-      "step_time": 4.244558566999331
+      "step_time": 25.519813745999954
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 52.0,
-      "completions/max_terminated_length": 52.0,
-      "completions/mean_length": 44.25,
-      "completions/mean_terminated_length": 44.25,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.009476900100708,
-      "epoch": 0.6991869918699187,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22703228890895844,
-      "kl": 1.2845549463236239e-05,
-      "learning_rate": 7.594438419578729e-07,
-      "loss": -0.005728684365749359,
-      "num_tokens": 360925.0,
-      "reward": 0.28028765320777893,
-      "reward_std": 0.2404259443283081,
-      "rewards/true_env_reward_fn/mean": 0.28028765320777893,
-      "rewards/true_env_reward_fn/std": 0.2404259443283081,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 65.95833587646484,
+      "completions/mean_terminated_length": 65.95833587646484,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3048341274261475,
+      "epoch": 2.097560975609756,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07558907568454742,
+      "kl": 1.8465078937879298e-05,
+      "learning_rate": 9.653257010912558e-07,
+      "loss": -0.015101172029972076,
+      "num_tokens": 2122829.0,
+      "reward": 0.4031229019165039,
+      "reward_std": 0.22406692802906036,
+      "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+      "rewards/true_env_reward_fn/std": 0.22406692802906036,
       "step": 86,
-      "step_time": 2.618181756000922
+      "step_time": 10.78625990699993
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 57.25,
-      "completions/mean_terminated_length": 57.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.1686812043190002,
-      "epoch": 0.7073170731707317,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.401248098583892e-05,
-      "kl": 1.2304412848607171e-05,
-      "learning_rate": 7.577670864066391e-07,
-      "loss": 6.143833388705389e-07,
-      "num_tokens": 362399.0,
-      "reward": 0.768503725528717,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.768503725528717,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 178.0,
+      "completions/max_terminated_length": 178.0,
+      "completions/mean_length": 82.125,
+      "completions/mean_terminated_length": 82.125,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3657839596271515,
+      "epoch": 2.1219512195121952,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07850468903779984,
+      "kl": 2.0332241774667636e-05,
+      "learning_rate": 9.637512255510474e-07,
+      "loss": 0.06651890277862549,
+      "num_tokens": 2151091.0,
+      "reward": 0.3940638303756714,
+      "reward_std": 0.2639860212802887,
+      "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+      "rewards/true_env_reward_fn/std": 0.2639860212802887,
       "step": 87,
-      "step_time": 3.34067542199773
+      "step_time": 13.604215705999877
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 71.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 59.125,
-      "completions/mean_terminated_length": 59.125,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.0876938998699188,
-      "epoch": 0.7154471544715447,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010135328921023756,
-      "kl": 1.3493038295564475e-05,
-      "learning_rate": 7.560582961509586e-07,
-      "loss": 6.750068450855906e-07,
-      "num_tokens": 365500.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 84.0,
+      "completions/mean_length": 68.125,
+      "completions/mean_terminated_length": 58.680850982666016,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.0381308495998383,
+      "epoch": 2.1463414634146343,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06663572043180466,
+      "kl": 2.4382573428738397e-05,
+      "learning_rate": 9.621431352937787e-07,
+      "loss": -0.08434788882732391,
+      "num_tokens": 2177281.0,
+      "reward": 0.40229034423828125,
+      "reward_std": 0.3266920745372772,
+      "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+      "rewards/true_env_reward_fn/std": 0.3266920745372772,
       "step": 88,
-      "step_time": 3.3087227100004384
+      "step_time": 32.408574500999975
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 61.25,
-      "completions/mean_terminated_length": 61.25,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.0288619995117188,
-      "epoch": 0.7235772357723578,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010261479474138469,
-      "kl": 1.3740621852775803e-05,
-      "learning_rate": 7.543176241970547e-07,
-      "loss": 6.875395683891838e-07,
-      "num_tokens": 369222.0,
-      "reward": 0.6557307243347168,
-      "reward_std": 0.2151959389448166,
-      "rewards/true_env_reward_fn/mean": 0.6557307243347168,
-      "rewards/true_env_reward_fn/std": 0.21519595384597778,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 60.8125,
+      "completions/mean_terminated_length": 60.8125,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.0944073796272278,
+      "epoch": 2.1707317073170733,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06779129058122635,
+      "kl": 2.3317856630455935e-05,
+      "learning_rate": 9.60501546880865e-07,
+      "loss": 0.019480882212519646,
+      "num_tokens": 2200208.0,
+      "reward": 0.5087729692459106,
+      "reward_std": 0.33071935176849365,
+      "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+      "rewards/true_env_reward_fn/std": 0.33071935176849365,
       "step": 89,
-      "step_time": 3.786183243999403
+      "step_time": 9.901715897000031
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 60.875,
-      "completions/mean_terminated_length": 60.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.1757304668426514,
-      "epoch": 0.7317073170731707,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2307090163230896,
-      "kl": 2.1445125639729667e-05,
-      "learning_rate": 7.525452264058595e-07,
-      "loss": 0.12042637169361115,
-      "num_tokens": 373465.0,
-      "reward": 0.4571714401245117,
-      "reward_std": 0.39374110102653503,
-      "rewards/true_env_reward_fn/mean": 0.4571714401245117,
-      "rewards/true_env_reward_fn/std": 0.39374107122421265,
+      "completions/max_length": 132.0,
+      "completions/max_terminated_length": 132.0,
+      "completions/mean_length": 65.4375,
+      "completions/mean_terminated_length": 65.4375,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.1683936715126038,
+      "epoch": 2.1951219512195124,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07440414279699326,
+      "kl": 1.8814549775925116e-05,
+      "learning_rate": 9.58826579301814e-07,
+      "loss": -0.03402159363031387,
+      "num_tokens": 2227789.0,
+      "reward": 0.40219685435295105,
+      "reward_std": 0.17851270735263824,
+      "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+      "rewards/true_env_reward_fn/std": 0.17851269245147705,
       "step": 90,
-      "step_time": 3.9787140030002774
+      "step_time": 11.152492722000034
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 71.0,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "entropy": 1.302090346813202,
-      "epoch": 0.7398373983739838,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16624286770820618,
-      "kl": 1.6463789506815374e-05,
-      "learning_rate": 7.507412614790579e-07,
-      "loss": -0.05975423753261566,
-      "num_tokens": 378029.0,
-      "reward": 0.3388232886791229,
-      "reward_std": 0.2467346489429474,
-      "rewards/true_env_reward_fn/mean": 0.3388232886791229,
-      "rewards/true_env_reward_fn/std": 0.24673466384410858,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 68.22917175292969,
+      "completions/mean_terminated_length": 68.22917175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.1628780961036682,
+      "epoch": 2.2195121951219514,
+      "frac_reward_zero_std": 0.8333333730697632,
+      "grad_norm": 0.03311198577284813,
+      "kl": 1.5094836498974473e-05,
+      "learning_rate": 9.57118353965601e-07,
+      "loss": 0.01087917946279049,
+      "num_tokens": 2252192.0,
+      "reward": 0.5357927083969116,
+      "reward_std": 0.18703003227710724,
+      "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+      "rewards/true_env_reward_fn/std": 0.18703003227710724,
       "step": 91,
-      "step_time": 3.9565000490001694
+      "step_time": 10.656350811000038
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 77.75,
-      "completions/mean_terminated_length": 77.75,
-      "completions/min_length": 56.0,
-      "completions/min_terminated_length": 56.0,
-      "entropy": 1.2768036723136902,
-      "epoch": 0.7479674796747967,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10557293146848679,
-      "kl": 1.2602345123013947e-05,
-      "learning_rate": 7.489058909448776e-07,
-      "loss": -0.023296140134334564,
-      "num_tokens": 380883.0,
-      "reward": 0.5399107933044434,
-      "reward_std": 0.26432597637176514,
-      "rewards/true_env_reward_fn/mean": 0.5399107933044434,
-      "rewards/true_env_reward_fn/std": 0.26432597637176514,
+      "completions/max_length": 121.0,
+      "completions/max_terminated_length": 121.0,
+      "completions/mean_length": 66.97917175292969,
+      "completions/mean_terminated_length": 66.97917175292969,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "entropy": 1.2369268834590912,
+      "epoch": 2.2439024390243905,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08958107978105545,
+      "kl": 2.8437810669856844e-05,
+      "learning_rate": 9.553769946918698e-07,
+      "loss": 0.005673397332429886,
+      "num_tokens": 2274199.0,
+      "reward": 0.5484694242477417,
+      "reward_std": 0.27515000104904175,
+      "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+      "rewards/true_env_reward_fn/std": 0.27515000104904175,
       "step": 92,
-      "step_time": 4.720347813999979
+      "step_time": 10.304143018000104
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 63.0,
-      "completions/max_terminated_length": 63.0,
-      "completions/mean_length": 49.0,
-      "completions/mean_terminated_length": 49.0,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.2670618891716003,
-      "epoch": 0.7560975609756098,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14662617444992065,
-      "kl": 1.812677373891347e-05,
-      "learning_rate": 7.470392791436244e-07,
-      "loss": -0.05785401538014412,
-      "num_tokens": 386095.0,
-      "reward": 0.30487915873527527,
-      "reward_std": 0.24597851932048798,
-      "rewards/true_env_reward_fn/mean": 0.30487915873527527,
-      "rewards/true_env_reward_fn/std": 0.24597853422164917,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 69.0625,
+      "completions/mean_terminated_length": 69.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.374023586511612,
+      "epoch": 2.2682926829268295,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.03936443477869034,
+      "kl": 1.8785845441016136e-05,
+      "learning_rate": 9.53602627701956e-07,
+      "loss": -0.01821933500468731,
+      "num_tokens": 2302818.0,
+      "reward": 0.3851678967475891,
+      "reward_std": 0.2433396279811859,
+      "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+      "rewards/true_env_reward_fn/std": 0.2433396428823471,
       "step": 93,
-      "step_time": 3.1318131530006212
+      "step_time": 13.589426085000014
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.1208478510379791,
-      "epoch": 0.7642276422764228,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011506211740197614,
-      "kl": 1.2571507795655634e-05,
-      "learning_rate": 7.451415932129691e-07,
-      "loss": 6.294373520177032e-07,
-      "num_tokens": 388335.0,
-      "reward": 0.7244763970375061,
-      "reward_std": 0.23028412461280823,
-      "rewards/true_env_reward_fn/mean": 0.7244763970375061,
-      "rewards/true_env_reward_fn/std": 0.23028412461280823,
+      "completions/max_length": 216.0,
+      "completions/max_terminated_length": 216.0,
+      "completions/mean_length": 68.9375,
+      "completions/mean_terminated_length": 68.9375,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.3004788756370544,
+      "epoch": 2.292682926829268,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06094004213809967,
+      "kl": 1.9176507976226276e-05,
+      "learning_rate": 9.517953816097395e-07,
+      "loss": 0.023817429319024086,
+      "num_tokens": 2325071.0,
+      "reward": 0.6004310846328735,
+      "reward_std": 0.23291133344173431,
+      "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+      "rewards/true_env_reward_fn/std": 0.23291133344173431,
       "step": 94,
-      "step_time": 3.6959203189999243
+      "step_time": 14.587356482000132
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 62.25,
-      "completions/mean_terminated_length": 62.25,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1998488903045654,
-      "epoch": 0.7723577235772358,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12171207368373871,
-      "kl": 1.6534771020815242e-05,
-      "learning_rate": 7.432130030729804e-07,
-      "loss": 0.05708106979727745,
-      "num_tokens": 393029.0,
-      "reward": 0.29566600918769836,
-      "reward_std": 0.2818882167339325,
-      "rewards/true_env_reward_fn/mean": 0.29566600918769836,
-      "rewards/true_env_reward_fn/std": 0.2818882167339325,
+      "completions/max_length": 237.0,
+      "completions/max_terminated_length": 237.0,
+      "completions/mean_length": 66.47917175292969,
+      "completions/mean_terminated_length": 66.47917175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.233375996351242,
+      "epoch": 2.317073170731707,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08650019764900208,
+      "kl": 1.89352349480032e-05,
+      "learning_rate": 9.499553874123212e-07,
+      "loss": 0.14434456825256348,
+      "num_tokens": 2347902.0,
+      "reward": 0.5542359948158264,
+      "reward_std": 0.18165862560272217,
+      "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+      "rewards/true_env_reward_fn/std": 0.18165862560272217,
       "step": 95,
-      "step_time": 4.322851452001487
+      "step_time": 14.689755582000089
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 49.375,
-      "completions/mean_terminated_length": 49.375,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.0649794340133667,
-      "epoch": 0.7804878048780488,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.2012680470943451,
-      "kl": 1.1399301456549438e-05,
-      "learning_rate": 7.412536814109106e-07,
-      "loss": -0.05478152632713318,
-      "num_tokens": 398112.0,
-      "reward": 0.23480799794197083,
-      "reward_std": 0.28209570050239563,
-      "rewards/true_env_reward_fn/mean": 0.23480799794197083,
-      "rewards/true_env_reward_fn/std": 0.282095730304718,
+      "completions/max_length": 173.0,
+      "completions/max_terminated_length": 173.0,
+      "completions/mean_length": 57.0625,
+      "completions/mean_terminated_length": 57.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.224440723657608,
+      "epoch": 2.341463414634146,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07974361628293991,
+      "kl": 1.838593607317307e-05,
+      "learning_rate": 9.480827784805278e-07,
+      "loss": 0.03995979577302933,
+      "num_tokens": 2361401.0,
+      "reward": 0.6956334114074707,
+      "reward_std": 0.185209721326828,
+      "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+      "rewards/true_env_reward_fn/std": 0.185209721326828,
       "step": 96,
-      "step_time": 3.4046103930013487
+      "step_time": 10.379233056999965
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 56.5,
-      "completions/mean_terminated_length": 56.5,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.4298859238624573,
-      "epoch": 0.7886178861788617,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2239074409008026,
-      "kl": 3.293174540885957e-05,
-      "learning_rate": 7.392638036657332e-07,
-      "loss": 0.09779056906700134,
-      "num_tokens": 402892.0,
-      "reward": 0.13796034455299377,
-      "reward_std": 0.22141560912132263,
-      "rewards/true_env_reward_fn/mean": 0.13796034455299377,
-      "rewards/true_env_reward_fn/std": 0.22141562402248383,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 60.25,
+      "completions/mean_terminated_length": 60.25,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.330334097146988,
+      "epoch": 2.3658536585365852,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08950946480035782,
+      "kl": 3.245086418246501e-05,
+      "learning_rate": 9.461776905492444e-07,
+      "loss": -0.03975849226117134,
+      "num_tokens": 2384437.0,
+      "reward": 0.49323582649230957,
+      "reward_std": 0.30376356840133667,
+      "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+      "rewards/true_env_reward_fn/std": 0.3037635385990143,
       "step": 97,
-      "step_time": 3.779275342998517
+      "step_time": 10.037491584999998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 52.0,
-      "completions/mean_terminated_length": 52.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2070425152778625,
-      "epoch": 0.7967479674796748,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.19742031395435333,
-      "kl": 1.4374184502230491e-05,
-      "learning_rate": 7.372435480124337e-07,
-      "loss": -0.006231316365301609,
-      "num_tokens": 408052.0,
-      "reward": 0.43320000171661377,
-      "reward_std": 0.05237230286002159,
-      "rewards/true_env_reward_fn/mean": 0.43320000171661377,
-      "rewards/true_env_reward_fn/std": 0.052372295409440994,
+      "completions/max_length": 163.0,
+      "completions/max_terminated_length": 163.0,
+      "completions/mean_length": 63.8125,
+      "completions/mean_terminated_length": 63.8125,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2434260249137878,
+      "epoch": 2.3902439024390243,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09637262672185898,
+      "kl": 3.597719251047238e-05,
+      "learning_rate": 9.442402617075764e-07,
+      "loss": 0.008840052410960197,
+      "num_tokens": 2409676.0,
+      "reward": 0.47345292568206787,
+      "reward_std": 0.3432519733905792,
+      "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+      "rewards/true_env_reward_fn/std": 0.34325194358825684,
       "step": 98,
-      "step_time": 3.1304682769987267
+      "step_time": 13.073343929999965
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 67.125,
-      "completions/mean_terminated_length": 67.125,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 0.9987849593162537,
-      "epoch": 0.8048780487804879,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.897383668227121e-05,
-      "kl": 1.2614300885616103e-05,
-      "learning_rate": 7.35193095346056e-07,
-      "loss": 6.314263600870618e-07,
-      "num_tokens": 409605.0,
-      "reward": 0.8541955947875977,
-      "reward_std": 0.09160846471786499,
-      "rewards/true_env_reward_fn/mean": 0.8541955947875977,
-      "rewards/true_env_reward_fn/std": 0.09160846471786499,
+      "completions/max_length": 101.0,
+      "completions/max_terminated_length": 101.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3253428936004639,
+      "epoch": 2.4146341463414633,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1034398227930069,
+      "kl": 3.889948129653931e-05,
+      "learning_rate": 9.422706323888396e-07,
+      "loss": 0.01636725291609764,
+      "num_tokens": 2433369.0,
+      "reward": 0.5016611218452454,
+      "reward_std": 0.3056275546550751,
+      "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+      "rewards/true_env_reward_fn/std": 0.3056274950504303,
       "step": 99,
-      "step_time": 4.13536422299876
+      "step_time": 9.465850557000067
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.0226224660873413,
-      "epoch": 0.8130081300813008,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20178858935832977,
-      "kl": 1.0500047665118473e-05,
-      "learning_rate": 7.331126292655044e-07,
-      "loss": -0.17970919609069824,
-      "num_tokens": 411488.0,
-      "reward": 0.6963247060775757,
-      "reward_std": 0.18840119242668152,
-      "rewards/true_env_reward_fn/mean": 0.6963247060775757,
-      "rewards/true_env_reward_fn/std": 0.1884012222290039,
+      "completions/max_length": 121.0,
+      "completions/max_terminated_length": 121.0,
+      "completions/mean_length": 62.125,
+      "completions/mean_terminated_length": 62.125,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 1.2484558820724487,
+      "epoch": 2.4390243902439024,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08669883012771606,
+      "kl": 1.584698543410923e-05,
+      "learning_rate": 9.402689453603814e-07,
+      "loss": 0.13139240443706512,
+      "num_tokens": 2458407.0,
+      "reward": 0.34693777561187744,
+      "reward_std": 0.35830602049827576,
+      "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+      "rewards/true_env_reward_fn/std": 0.35830605030059814,
       "step": 100,
-      "step_time": 3.7544156769981782
+      "step_time": 11.33050741000011
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 59.0,
-      "completions/mean_terminated_length": 59.0,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2509461045265198,
-      "epoch": 0.8211382113821138,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22887632250785828,
-      "kl": 2.1612477212329395e-05,
-      "learning_rate": 7.310023360571047e-07,
-      "loss": 0.025605827569961548,
-      "num_tokens": 414080.0,
-      "reward": 0.588032603263855,
-      "reward_std": 0.11032751202583313,
-      "rewards/true_env_reward_fn/mean": 0.588032603263855,
-      "rewards/true_env_reward_fn/std": 0.11032749712467194,
+      "completions/max_length": 225.0,
+      "completions/max_terminated_length": 225.0,
+      "completions/mean_length": 68.77083587646484,
+      "completions/mean_terminated_length": 68.77083587646484,
+      "completions/min_length": 17.0,
+      "completions/min_terminated_length": 17.0,
+      "entropy": 1.2351897060871124,
+      "epoch": 2.4634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06004978343844414,
+      "kl": 2.0037293097630027e-05,
+      "learning_rate": 9.382353457132317e-07,
+      "loss": -0.04131116345524788,
+      "num_tokens": 2483052.0,
+      "reward": 0.38015443086624146,
+      "reward_std": 0.34710174798965454,
+      "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+      "rewards/true_env_reward_fn/std": 0.34710174798965454,
       "step": 101,
-      "step_time": 3.625197022998691
+      "step_time": 16.478299477000064
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 176.0,
-      "completions/max_terminated_length": 176.0,
-      "completions/mean_length": 95.375,
-      "completions/mean_terminated_length": 95.375,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.5443179607391357,
-      "epoch": 0.8292682926829268,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10614532232284546,
-      "kl": 1.492139062975184e-05,
-      "learning_rate": 7.28862404677924e-07,
-      "loss": 0.06531564146280289,
-      "num_tokens": 419835.0,
-      "reward": 0.07074306160211563,
-      "reward_std": 0.2918013632297516,
-      "rewards/true_env_reward_fn/mean": 0.07074306160211563,
-      "rewards/true_env_reward_fn/std": 0.2918013632297516,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3605049848556519,
+      "epoch": 2.4878048780487805,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09010742604732513,
+      "kl": 3.810847783825011e-05,
+      "learning_rate": 9.361699808515875e-07,
+      "loss": 0.038716960698366165,
+      "num_tokens": 2510193.0,
+      "reward": 0.3458574712276459,
+      "reward_std": 0.30283215641975403,
+      "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+      "rewards/true_env_reward_fn/std": 0.30283215641975403,
       "step": 102,
-      "step_time": 7.796810614998321
+      "step_time": 11.344593008000174
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 101.0,
       "completions/max_terminated_length": 101.0,
-      "completions/mean_length": 55.25,
-      "completions/mean_terminated_length": 55.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.3223788738250732,
-      "epoch": 0.8373983739837398,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2421368807554245,
-      "kl": 3.265505938543356e-05,
-      "learning_rate": 7.266930267388503e-07,
-      "loss": -0.07752113044261932,
-      "num_tokens": 422773.0,
-      "reward": 0.33568501472473145,
-      "reward_std": 0.2780380845069885,
-      "rewards/true_env_reward_fn/mean": 0.33568501472473145,
-      "rewards/true_env_reward_fn/std": 0.2780380845069885,
+      "completions/mean_length": 65.22917175292969,
+      "completions/mean_terminated_length": 65.22917175292969,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.19815993309021,
+      "epoch": 2.5121951219512195,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07395386695861816,
+      "kl": 2.6301003344997298e-05,
+      "learning_rate": 9.340730004821265e-07,
+      "loss": 0.01458972692489624,
+      "num_tokens": 2529212.0,
+      "reward": 0.5586616396903992,
+      "reward_std": 0.20150764286518097,
+      "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+      "rewards/true_env_reward_fn/std": 0.20150764286518097,
       "step": 103,
-      "step_time": 4.313938073000827
+      "step_time": 8.135681302999728
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2900,26 +2900,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 67.25,
-      "completions/mean_terminated_length": 67.25,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.3332037329673767,
-      "epoch": 0.8455284552845529,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13883370161056519,
-      "kl": 2.1224042484391248e-05,
-      "learning_rate": 7.244943964874369e-07,
-      "loss": 0.021739646792411804,
-      "num_tokens": 426507.0,
-      "reward": 0.40595096349716187,
-      "reward_std": 0.2035457342863083,
-      "rewards/true_env_reward_fn/mean": 0.40595096349716187,
-      "rewards/true_env_reward_fn/std": 0.2035457193851471,
+      "completions/max_length": 166.0,
+      "completions/max_terminated_length": 166.0,
+      "completions/mean_length": 73.89583587646484,
+      "completions/mean_terminated_length": 73.89583587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2152214348316193,
+      "epoch": 2.5365853658536586,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08900879323482513,
+      "kl": 3.066915814997628e-05,
+      "learning_rate": 9.31944556603157e-07,
+      "loss": 0.08802390843629837,
+      "num_tokens": 2557007.0,
+      "reward": 0.4009184241294861,
+      "reward_std": 0.32733896374702454,
+      "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+      "rewards/true_env_reward_fn/std": 0.3273389935493469,
       "step": 104,
-      "step_time": 4.155937195999286
+      "step_time": 15.185034105999875
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 68.39583587646484,
+      "completions/mean_terminated_length": 68.39583587646484,
       "completions/min_length": 31.0,
       "completions/min_terminated_length": 31.0,
-      "entropy": 1.3391229510307312,
-      "epoch": 0.8536585365853658,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24205009639263153,
-      "kl": 2.5022183763212524e-05,
-      "learning_rate": 7.222667107905085e-07,
-      "loss": 0.06330433487892151,
-      "num_tokens": 429010.0,
-      "reward": 0.3355163037776947,
-      "reward_std": 0.2902730703353882,
-      "rewards/true_env_reward_fn/mean": 0.3355163037776947,
-      "rewards/true_env_reward_fn/std": 0.29027310013771057,
+      "entropy": 1.2849501073360443,
+      "epoch": 2.5609756097560976,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0703769251704216,
+      "kl": 1.9505746195136453e-05,
+      "learning_rate": 9.297848034936005e-07,
+      "loss": 0.036192238330841064,
+      "num_tokens": 2581170.0,
+      "reward": 0.4875798225402832,
+      "reward_std": 0.16742677986621857,
+      "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+      "rewards/true_env_reward_fn/std": 0.16742677986621857,
       "step": 105,
-      "step_time": 3.808478789000219
+      "step_time": 9.588520330999927
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.1864720582962036,
-      "epoch": 0.8617886178861789,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13473568856716156,
-      "kl": 1.4212585938366828e-05,
-      "learning_rate": 7.200101691165338e-07,
-      "loss": -0.020715661346912384,
-      "num_tokens": 432403.0,
-      "reward": 0.4871198534965515,
-      "reward_std": 0.15407639741897583,
-      "rewards/true_env_reward_fn/mean": 0.4871198534965515,
-      "rewards/true_env_reward_fn/std": 0.15407641232013702,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 66.95833587646484,
+      "completions/mean_terminated_length": 66.95833587646484,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.2687023878097534,
+      "epoch": 2.5853658536585367,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08327006548643112,
+      "kl": 2.3203040655062068e-05,
+      "learning_rate": 9.275938977018081e-07,
+      "loss": 0.003695126622915268,
+      "num_tokens": 2609408.0,
+      "reward": 0.40928101539611816,
+      "reward_std": 0.10633077472448349,
+      "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+      "rewards/true_env_reward_fn/std": 0.1063307598233223,
       "step": 106,
-      "step_time": 4.240638332001254
+      "step_time": 14.60399662399982
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 48.0,
-      "completions/mean_terminated_length": 48.0,
-      "completions/min_length": 24.0,
-      "completions/min_terminated_length": 24.0,
-      "entropy": 1.0669284462928772,
-      "epoch": 0.8699186991869918,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14131899178028107,
-      "kl": 1.5787159554747632e-05,
-      "learning_rate": 7.177249735177651e-07,
-      "loss": 0.03678784519433975,
-      "num_tokens": 435995.0,
-      "reward": 0.5010770559310913,
-      "reward_std": 0.48966261744499207,
-      "rewards/true_env_reward_fn/mean": 0.5010770559310913,
-      "rewards/true_env_reward_fn/std": 0.48966261744499207,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 79.79167175292969,
+      "completions/mean_terminated_length": 79.79167175292969,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.1623006761074066,
+      "epoch": 2.6097560975609757,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07009758800268173,
+      "kl": 2.6010310648416635e-05,
+      "learning_rate": 9.253719980342134e-07,
+      "loss": -0.025412028655409813,
+      "num_tokens": 2641446.0,
+      "reward": 0.29606160521507263,
+      "reward_std": 0.3615049123764038,
+      "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+      "rewards/true_env_reward_fn/std": 0.3615049123764038,
       "step": 107,
-      "step_time": 3.3587191269998584
+      "step_time": 20.100954443999854
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 71.875,
-      "completions/mean_terminated_length": 71.875,
-      "completions/min_length": 57.0,
-      "completions/min_terminated_length": 57.0,
-      "entropy": 1.304731547832489,
-      "epoch": 0.8780487804878049,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.380985673284158e-05,
-      "kl": 1.3128728824085556e-05,
-      "learning_rate": 7.154113286121462e-07,
-      "loss": 6.494262834166875e-07,
-      "num_tokens": 442094.0,
-      "reward": 0.4055500030517578,
-      "reward_std": 0.052258480340242386,
-      "rewards/true_env_reward_fn/mean": 0.4055500030517578,
-      "rewards/true_env_reward_fn/std": 0.052258484065532684,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 60.6875,
+      "completions/mean_terminated_length": 60.6875,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 1.2318958044052124,
+      "epoch": 2.6341463414634148,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09370094537734985,
+      "kl": 2.169116805816884e-05,
+      "learning_rate": 9.23119265543822e-07,
+      "loss": -0.009763844311237335,
+      "num_tokens": 2659695.0,
+      "reward": 0.5309837460517883,
+      "reward_std": 0.1692933589220047,
+      "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+      "rewards/true_env_reward_fn/std": 0.1692933589220047,
       "step": 108,
-      "step_time": 4.337008413998774
+      "step_time": 8.304149297999857
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2324069738388062,
-      "epoch": 0.8861788617886179,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13588950037956238,
-      "kl": 1.3448377558233915e-05,
-      "learning_rate": 7.130694415649912e-07,
-      "loss": 1.0952353477478027e-06,
-      "num_tokens": 447226.0,
-      "reward": 0.20854972302913666,
-      "reward_std": 0.06059705466032028,
-      "rewards/true_env_reward_fn/mean": 0.20854972302913666,
-      "rewards/true_env_reward_fn/std": 0.06059705838561058,
+      "completions/max_length": 117.0,
+      "completions/max_terminated_length": 117.0,
+      "completions/mean_length": 66.29167175292969,
+      "completions/mean_terminated_length": 66.29167175292969,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.248624861240387,
+      "epoch": 2.658536585365854,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09214548021554947,
+      "kl": 5.0202284000988584e-05,
+      "learning_rate": 9.208358635185372e-07,
+      "loss": 0.0672653466463089,
+      "num_tokens": 2691005.0,
+      "reward": 0.306609183549881,
+      "reward_std": 0.24702024459838867,
+      "rewards/true_env_reward_fn/mean": 0.306609183549881,
+      "rewards/true_env_reward_fn/std": 0.24702024459838867,
       "step": 109,
-      "step_time": 3.2976038649994734
+      "step_time": 11.260021517000041
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 67.625,
-      "completions/mean_terminated_length": 67.625,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1567262411117554,
-      "epoch": 0.8943089430894309,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.736967720324174e-05,
-      "kl": 1.2838129805459175e-05,
-      "learning_rate": 7.106995220704342e-07,
-      "loss": 6.425898391171359e-07,
-      "num_tokens": 450359.0,
-      "reward": 0.7316612601280212,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.7316612601280212,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 294.0,
+      "completions/max_terminated_length": 294.0,
+      "completions/mean_length": 77.64583587646484,
+      "completions/mean_terminated_length": 77.64583587646484,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 1.2180723249912262,
+      "epoch": 2.682926829268293,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08635839074850082,
+      "kl": 3.741631462617079e-05,
+      "learning_rate": 9.185219574693241e-07,
+      "loss": 0.06331576406955719,
+      "num_tokens": 2717196.0,
+      "reward": 0.5423221588134766,
+      "reward_std": 0.3347312808036804,
+      "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+      "rewards/true_env_reward_fn/std": 0.3347312808036804,
       "step": 110,
-      "step_time": 4.067084037998939
+      "step_time": 22.80178854000019
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 100.0,
       "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 65.0,
-      "completions/mean_terminated_length": 65.0,
-      "completions/min_length": 12.0,
-      "completions/min_terminated_length": 12.0,
-      "entropy": 1.496058464050293,
-      "epoch": 0.9024390243902439,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18041981756687164,
-      "kl": 1.6616825632809196e-05,
-      "learning_rate": 7.083017823326532e-07,
-      "loss": 0.0269068144261837,
-      "num_tokens": 453583.0,
-      "reward": 0.5647265911102295,
-      "reward_std": 0.1507105529308319,
-      "rewards/true_env_reward_fn/mean": 0.5647265911102295,
-      "rewards/true_env_reward_fn/std": 0.1507105529308319,
+      "completions/mean_length": 55.5,
+      "completions/mean_terminated_length": 55.5,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2095272839069366,
+      "epoch": 2.7073170731707314,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09482823312282562,
+      "kl": 3.445757738518296e-05,
+      "learning_rate": 9.161777151182135e-07,
+      "loss": -0.007809684611856937,
+      "num_tokens": 2739924.0,
+      "reward": 0.4738404154777527,
+      "reward_std": 0.2762244939804077,
+      "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+      "rewards/true_env_reward_fn/std": 0.2762244939804077,
       "step": 111,
-      "step_time": 4.347732382997492
+      "step_time": 9.663163859000178
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 108.0,
-      "completions/max_terminated_length": 108.0,
-      "completions/mean_length": 67.75,
-      "completions/mean_terminated_length": 67.75,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.4196155667304993,
-      "epoch": 0.9105691056910569,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18451112508773804,
-      "kl": 2.1803500203532167e-05,
-      "learning_rate": 7.058764370468698e-07,
-      "loss": 0.1650262475013733,
-      "num_tokens": 456773.0,
-      "reward": 0.6907394528388977,
-      "reward_std": 0.1393815129995346,
-      "rewards/true_env_reward_fn/mean": 0.6907394528388977,
-      "rewards/true_env_reward_fn/std": 0.1393815129995346,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 72.79167175292969,
+      "completions/mean_terminated_length": 72.79167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.339354246854782,
+      "epoch": 2.7317073170731705,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09729615598917007,
+      "kl": 4.7237631861207774e-05,
+      "learning_rate": 9.138033063861434e-07,
+      "loss": 0.0440620519220829,
+      "num_tokens": 2763226.0,
+      "reward": 0.4624016284942627,
+      "reward_std": 0.2299472838640213,
+      "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+      "rewards/true_env_reward_fn/std": 0.2299472540616989,
       "step": 112,
-      "step_time": 4.627644968999448
+      "step_time": 9.903081222999617
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 61.125,
-      "completions/mean_terminated_length": 61.125,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2243221998214722,
-      "epoch": 0.9186991869918699,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13331371545791626,
-      "kl": 1.548633599668392e-05,
-      "learning_rate": 7.034237033801247e-07,
-      "loss": 0.039844345301389694,
-      "num_tokens": 462714.0,
-      "reward": 0.21676866710186005,
-      "reward_std": 0.26559779047966003,
-      "rewards/true_env_reward_fn/mean": 0.21676866710186005,
-      "rewards/true_env_reward_fn/std": 0.26559779047966003,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 62.5625,
+      "completions/mean_terminated_length": 62.5625,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2139239311218262,
+      "epoch": 2.7560975609756095,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09817806631326675,
+      "kl": 5.5064369917090517e-05,
+      "learning_rate": 9.113989033806433e-07,
+      "loss": 0.03889988735318184,
+      "num_tokens": 2788677.0,
+      "reward": 0.3767920434474945,
+      "reward_std": 0.3360261619091034,
+      "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+      "rewards/true_env_reward_fn/std": 0.3360261619091034,
       "step": 113,
-      "step_time": 3.8455466220002563
+      "step_time": 10.101770388999284
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 73.0,
-      "completions/mean_terminated_length": 73.0,
-      "completions/min_length": 58.0,
-      "completions/min_terminated_length": 58.0,
-      "entropy": 1.325823724269867,
-      "epoch": 0.926829268292683,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1866220384836197,
-      "kl": 1.8801019905367866e-05,
-      "learning_rate": 7.009438009518325e-07,
-      "loss": 0.06504581868648529,
-      "num_tokens": 465994.0,
-      "reward": 0.5194582939147949,
-      "reward_std": 0.2796703577041626,
-      "rewards/true_env_reward_fn/mean": 0.5194582939147949,
-      "rewards/true_env_reward_fn/std": 0.2796703577041626,
+      "completions/max_length": 236.0,
+      "completions/max_terminated_length": 236.0,
+      "completions/mean_length": 69.29167175292969,
+      "completions/mean_terminated_length": 69.29167175292969,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.2278488278388977,
+      "epoch": 2.7804878048780486,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09011705964803696,
+      "kl": 3.285038519607042e-05,
+      "learning_rate": 9.089646803833588e-07,
+      "loss": 0.03598163276910782,
+      "num_tokens": 2812139.0,
+      "reward": 0.5151569843292236,
+      "reward_std": 0.24896851181983948,
+      "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+      "rewards/true_env_reward_fn/std": 0.24896851181983948,
       "step": 114,
-      "step_time": 4.151028698999653
+      "step_time": 17.633509853000305
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.1521879434585571,
-      "epoch": 0.9349593495934959,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14320029318332672,
-      "kl": 1.2749982033710694e-05,
-      "learning_rate": 6.98436951814117e-07,
-      "loss": 0.03685300797224045,
-      "num_tokens": 468615.0,
-      "reward": 0.5399107933044434,
-      "reward_std": 0.26432597637176514,
-      "rewards/true_env_reward_fn/mean": 0.5399107933044434,
-      "rewards/true_env_reward_fn/std": 0.26432597637176514,
+      "completions/max_length": 151.0,
+      "completions/max_terminated_length": 151.0,
+      "completions/mean_length": 61.97916793823242,
+      "completions/mean_terminated_length": 61.97916793823242,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.1842038929462433,
+      "epoch": 2.8048780487804876,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07707802206277847,
+      "kl": 4.165519931120798e-05,
+      "learning_rate": 9.065008138374188e-07,
+      "loss": 0.03350803256034851,
+      "num_tokens": 2835354.0,
+      "reward": 0.4122808873653412,
+      "reward_std": 0.27231934666633606,
+      "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+      "rewards/true_env_reward_fn/std": 0.27231931686401367,
       "step": 115,
-      "step_time": 3.6973990600017714
+      "step_time": 12.307247350000125
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.201507806777954,
-      "epoch": 0.943089430894309,
+      "completions/max_length": 232.0,
+      "completions/max_terminated_length": 232.0,
+      "completions/mean_length": 83.64583587646484,
+      "completions/mean_terminated_length": 83.64583587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3357974886894226,
+      "epoch": 2.8292682926829267,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14264807105064392,
-      "kl": 2.6679515940486453e-05,
-      "learning_rate": 6.959033804319283e-07,
-      "loss": -0.023484818637371063,
-      "num_tokens": 471647.0,
-      "reward": 0.41836902499198914,
-      "reward_std": 0.3116860091686249,
-      "rewards/true_env_reward_fn/mean": 0.41836902499198914,
-      "rewards/true_env_reward_fn/std": 0.3116860091686249,
+      "grad_norm": 0.06329861283302307,
+      "kl": 1.8487026636648807e-05,
+      "learning_rate": 9.040074823346464e-07,
+      "loss": 0.030132077634334564,
+      "num_tokens": 2859017.0,
+      "reward": 0.5723411440849304,
+      "reward_std": 0.21183526515960693,
+      "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+      "rewards/true_env_reward_fn/std": 0.21183528006076813,
       "step": 116,
-      "step_time": 3.1295652919998247
+      "step_time": 14.468690254000194
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 62.375,
-      "completions/mean_terminated_length": 62.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2834057807922363,
-      "epoch": 0.9512195121951219,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.810227154754102e-05,
-      "kl": 1.4841665233689127e-05,
-      "learning_rate": 6.933433136629443e-07,
-      "loss": 7.425555850204546e-07,
-      "num_tokens": 474682.0,
-      "reward": 0.6203632950782776,
-      "reward_std": 0.11898252367973328,
-      "rewards/true_env_reward_fn/mean": 0.6203632950782776,
-      "rewards/true_env_reward_fn/std": 0.11898253113031387,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 70.35417175292969,
+      "completions/mean_terminated_length": 70.35417175292969,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 1.1871840357780457,
+      "epoch": 2.8536585365853657,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06702237576246262,
+      "kl": 2.817388576659141e-05,
+      "learning_rate": 9.014848666026138e-07,
+      "loss": 0.00200769305229187,
+      "num_tokens": 2889050.0,
+      "reward": 0.3500348925590515,
+      "reward_std": 0.30559059977531433,
+      "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+      "rewards/true_env_reward_fn/std": 0.3055906295776367,
       "step": 117,
-      "step_time": 3.4368692790012574
+      "step_time": 11.849063975999798
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 63.625,
-      "completions/mean_terminated_length": 63.625,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.191932499408722,
-      "epoch": 0.959349593495935,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21204856038093567,
-      "kl": 3.64198385796044e-05,
-      "learning_rate": 6.907569807372574e-07,
-      "loss": -0.001312553882598877,
-      "num_tokens": 477027.0,
-      "reward": 0.5300568342208862,
-      "reward_std": 0.2945883274078369,
-      "rewards/true_env_reward_fn/mean": 0.5300568342208862,
-      "rewards/true_env_reward_fn/std": 0.2945883274078369,
+      "completions/max_length": 239.0,
+      "completions/max_terminated_length": 239.0,
+      "completions/mean_length": 79.4375,
+      "completions/mean_terminated_length": 79.4375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2978005111217499,
+      "epoch": 2.8780487804878048,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0668371245265007,
+      "kl": 3.733048197318567e-05,
+      "learning_rate": 8.989331494915416e-07,
+      "loss": -0.04680684953927994,
+      "num_tokens": 2917335.0,
+      "reward": 0.32068905234336853,
+      "reward_std": 0.30586519837379456,
+      "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+      "rewards/true_env_reward_fn/std": 0.30586519837379456,
       "step": 118,
-      "step_time": 3.8569856240010267
+      "step_time": 16.597334930000216
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 190.0,
-      "completions/max_terminated_length": 190.0,
-      "completions/mean_length": 96.5,
-      "completions/mean_terminated_length": 96.5,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.2401175498962402,
-      "epoch": 0.967479674796748,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011377666669432074,
-      "kl": 1.3742283954343293e-05,
-      "learning_rate": 6.881446132368494e-07,
-      "loss": 6.866695230201003e-07,
-      "num_tokens": 481999.0,
-      "reward": 0.5930472612380981,
-      "reward_std": 0.14818456768989563,
-      "rewards/true_env_reward_fn/mean": 0.5930472612380981,
-      "rewards/true_env_reward_fn/std": 0.14818456768989563,
+      "completions/max_length": 164.0,
+      "completions/max_terminated_length": 164.0,
+      "completions/mean_length": 69.10417175292969,
+      "completions/mean_terminated_length": 69.10417175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2123413980007172,
+      "epoch": 2.902439024390244,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.074281245470047,
+      "kl": 3.654057309177006e-05,
+      "learning_rate": 8.963525159610464e-07,
+      "loss": 0.0373641662299633,
+      "num_tokens": 2938004.0,
+      "reward": 0.556015133857727,
+      "reward_std": 0.22209766507148743,
+      "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+      "rewards/true_env_reward_fn/std": 0.22209767997264862,
       "step": 119,
-      "step_time": 8.09440958399864
+      "step_time": 11.729475523999554
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 66.125,
-      "completions/mean_terminated_length": 66.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.230682611465454,
-      "epoch": 0.975609756097561,
+      "completions/max_length": 135.0,
+      "completions/max_terminated_length": 135.0,
+      "completions/mean_length": 66.08333587646484,
+      "completions/mean_terminated_length": 66.08333587646484,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.3360244035720825,
+      "epoch": 2.926829268292683,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22175048291683197,
-      "kl": 1.2522132237791084e-05,
-      "learning_rate": 6.855064450748555e-07,
-      "loss": -0.04083740711212158,
-      "num_tokens": 490884.0,
-      "reward": 0.13476666808128357,
-      "reward_std": 0.2987530529499054,
-      "rewards/true_env_reward_fn/mean": 0.13476666808128357,
-      "rewards/true_env_reward_fn/std": 0.2987530827522278,
+      "grad_norm": 0.09382818639278412,
+      "kl": 3.540705620252993e-05,
+      "learning_rate": 8.937431530667327e-07,
+      "loss": 0.057918041944503784,
+      "num_tokens": 2966976.0,
+      "reward": 0.3999954164028168,
+      "reward_std": 0.2351321578025818,
+      "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+      "rewards/true_env_reward_fn/std": 0.23513217270374298,
       "step": 120,
-      "step_time": 4.678523641001448
+      "step_time": 11.503627788000358
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 63.5,
-      "completions/mean_terminated_length": 63.5,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2799639105796814,
-      "epoch": 0.983739837398374,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19926966726779938,
-      "kl": 1.7022688552970067e-05,
-      "learning_rate": 6.828427124746189e-07,
-      "loss": -0.010804429650306702,
-      "num_tokens": 496404.0,
-      "reward": 0.24633333086967468,
-      "reward_std": 0.2454334795475006,
-      "rewards/true_env_reward_fn/mean": 0.24633333086967468,
-      "rewards/true_env_reward_fn/std": 0.2454334795475006,
+      "completions/max_length": 98.0,
+      "completions/max_terminated_length": 98.0,
+      "completions/mean_length": 58.97916793823242,
+      "completions/mean_terminated_length": 58.97916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2053601145744324,
+      "epoch": 2.951219512195122,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07024823874235153,
+      "kl": 3.6033439755556174e-05,
+      "learning_rate": 8.911052499466356e-07,
+      "loss": 0.04910843074321747,
+      "num_tokens": 2987391.0,
+      "reward": 0.5365906953811646,
+      "reward_std": 0.19872017204761505,
+      "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+      "rewards/true_env_reward_fn/std": 0.19872015714645386,
       "step": 121,
-      "step_time": 3.98071062300005
+      "step_time": 8.728293746000418
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 46.875,
-      "completions/mean_terminated_length": 46.875,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.3840235471725464,
-      "epoch": 0.991869918699187,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24853822588920593,
-      "kl": 3.688259130285587e-05,
-      "learning_rate": 6.801536539485403e-07,
-      "loss": 0.10205884277820587,
-      "num_tokens": 499767.0,
-      "reward": 0.3045905530452728,
-      "reward_std": 0.262839138507843,
-      "rewards/true_env_reward_fn/mean": 0.3045905530452728,
-      "rewards/true_env_reward_fn/std": 0.262839138507843,
+      "completions/max_length": 219.0,
+      "completions/max_terminated_length": 219.0,
+      "completions/mean_length": 70.8125,
+      "completions/mean_terminated_length": 70.8125,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.296659678220749,
+      "epoch": 2.975609756097561,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0607762485742569,
+      "kl": 3.175417168677086e-05,
+      "learning_rate": 8.884389978075097e-07,
+      "loss": -0.040265124291181564,
+      "num_tokens": 3009358.0,
+      "reward": 0.49613699316978455,
+      "reward_std": 0.2080756276845932,
+      "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+      "rewards/true_env_reward_fn/std": 0.2080756276845932,
       "step": 122,
-      "step_time": 3.3792565210005705
+      "step_time": 15.51957702100026
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 50.375,
-      "completions/mean_terminated_length": 50.375,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.2064164280891418,
-      "epoch": 1.0,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1365528553724289,
-      "kl": 2.434901080050622e-05,
-      "learning_rate": 6.774395102767203e-07,
-      "loss": -0.03472680225968361,
-      "num_tokens": 504906.0,
-      "reward": 0.2722649872303009,
-      "reward_std": 0.2922348082065582,
-      "rewards/true_env_reward_fn/mean": 0.2722649872303009,
-      "rewards/true_env_reward_fn/std": 0.2922348082065582,
+      "completions/max_length": 93.0,
+      "completions/max_terminated_length": 93.0,
+      "completions/mean_length": 59.4375,
+      "completions/mean_terminated_length": 59.4375,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.2992768585681915,
+      "epoch": 3.0,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09044087678194046,
+      "kl": 6.319149179034866e-05,
+      "learning_rate": 8.857445899109715e-07,
+      "loss": -0.030733143910765648,
+      "num_tokens": 3035563.0,
+      "reward": 0.34821078181266785,
+      "reward_std": 0.2354777753353119,
+      "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+      "rewards/true_env_reward_fn/std": 0.23547779023647308,
       "step": 123,
-      "step_time": 3.0233660449994204
+      "step_time": 8.471463828000196
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2819936871528625,
-      "epoch": 1.008130081300813,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00023045104171615094,
-      "kl": 2.2608143808611203e-05,
-      "learning_rate": 6.747005244854004e-07,
-      "loss": 1.1284330412308918e-06,
-      "num_tokens": 508329.0,
-      "reward": 0.3149532079696655,
-      "reward_std": 0.4275679290294647,
-      "rewards/true_env_reward_fn/mean": 0.3149532079696655,
-      "rewards/true_env_reward_fn/std": 0.4275679886341095,
+      "completions/max_length": 193.0,
+      "completions/max_terminated_length": 193.0,
+      "completions/mean_length": 69.64583587646484,
+      "completions/mean_terminated_length": 69.64583587646484,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.2003771364688873,
+      "epoch": 3.024390243902439,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08278124779462814,
+      "kl": 2.7146410047862446e-05,
+      "learning_rate": 8.83022221559489e-07,
+      "loss": 0.02903313934803009,
+      "num_tokens": 3056090.0,
+      "reward": 0.5313810110092163,
+      "reward_std": 0.18604923784732819,
+      "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+      "rewards/true_env_reward_fn/std": 0.18604923784732819,
       "step": 124,
-      "step_time": 4.01701365199915
+      "step_time": 13.438758649999727
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 157.0,
-      "completions/max_terminated_length": 157.0,
-      "completions/mean_length": 80.875,
-      "completions/mean_terminated_length": 80.875,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.1542360186576843,
-      "epoch": 1.016260162601626,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.232049003709108e-05,
-      "kl": 1.291002809011843e-05,
-      "learning_rate": 6.719369418252023e-07,
-      "loss": 6.488799613180163e-07,
-      "num_tokens": 515076.0,
-      "reward": 0.4841846525669098,
-      "reward_std": 0.12780573964118958,
-      "rewards/true_env_reward_fn/mean": 0.4841846525669098,
-      "rewards/true_env_reward_fn/std": 0.12780575454235077,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 63.625,
+      "completions/mean_terminated_length": 63.625,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2338614165782928,
+      "epoch": 3.048780487804878,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06770245730876923,
+      "kl": 2.570231345089269e-05,
+      "learning_rate": 8.802720900822269e-07,
+      "loss": 0.0324365571141243,
+      "num_tokens": 3080424.0,
+      "reward": 0.44920405745506287,
+      "reward_std": 0.206027552485466,
+      "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+      "rewards/true_env_reward_fn/std": 0.2060275673866272,
       "step": 125,
-      "step_time": 7.240956699999515
+      "step_time": 8.654177170999901
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 56.375,
-      "completions/mean_terminated_length": 56.375,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2594389915466309,
-      "epoch": 1.024390243902439,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13091468811035156,
-      "kl": 1.751603304001037e-05,
-      "learning_rate": 6.691490097491675e-07,
-      "loss": -0.033413223922252655,
-      "num_tokens": 520279.0,
-      "reward": 0.28095200657844543,
-      "reward_std": 0.21837711334228516,
-      "rewards/true_env_reward_fn/mean": 0.28095200657844543,
-      "rewards/true_env_reward_fn/std": 0.21837712824344635,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 70.10417175292969,
+      "completions/mean_terminated_length": 70.10417175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.1446799635887146,
+      "epoch": 3.073170731707317,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.060568179935216904,
+      "kl": 3.4001183394138934e-05,
+      "learning_rate": 8.774943948207425e-07,
+      "loss": -0.009533079341053963,
+      "num_tokens": 3100469.0,
+      "reward": 0.5536229610443115,
+      "reward_std": 0.29822590947151184,
+      "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+      "rewards/true_env_reward_fn/std": 0.29822590947151184,
       "step": 126,
-      "step_time": 3.355879656997786
+      "step_time": 10.513378469000145
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 136.0,
-      "completions/max_terminated_length": 136.0,
-      "completions/mean_length": 77.875,
-      "completions/mean_terminated_length": 77.875,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.3044686317443848,
-      "epoch": 1.032520325203252,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12953205406665802,
-      "kl": 1.8700401597016025e-05,
-      "learning_rate": 6.663369778906008e-07,
-      "loss": 0.03562816232442856,
-      "num_tokens": 524582.0,
-      "reward": 0.4330660402774811,
-      "reward_std": 0.4592672288417816,
-      "rewards/true_env_reward_fn/mean": 0.4330660402774811,
-      "rewards/true_env_reward_fn/std": 0.459267258644104,
+      "completions/max_length": 348.0,
+      "completions/max_terminated_length": 348.0,
+      "completions/mean_length": 75.5,
+      "completions/mean_terminated_length": 75.5,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3296749591827393,
+      "epoch": 3.097560975609756,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06767670065164566,
+      "kl": 4.0856727537175175e-05,
+      "learning_rate": 8.746893371145365e-07,
+      "loss": -0.023851895704865456,
+      "num_tokens": 3127557.0,
+      "reward": 0.3543795943260193,
+      "reward_std": 0.3506966233253479,
+      "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+      "rewards/true_env_reward_fn/std": 0.3506965935230255,
       "step": 127,
-      "step_time": 5.965807722999671
+      "step_time": 23.20779430600078
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 117.0,
-      "completions/max_terminated_length": 117.0,
-      "completions/mean_length": 75.375,
-      "completions/mean_terminated_length": 75.375,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.1742327809333801,
-      "epoch": 1.040650406504065,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14993594586849213,
-      "kl": 1.1459212601039326e-05,
-      "learning_rate": 6.635010980407174e-07,
-      "loss": 0.03646668791770935,
-      "num_tokens": 526213.0,
-      "reward": 0.7185037136077881,
-      "reward_std": 0.1414213478565216,
-      "rewards/true_env_reward_fn/mean": 0.7185037136077881,
-      "rewards/true_env_reward_fn/std": 0.1414213627576828,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 62.520835876464844,
+      "completions/mean_terminated_length": 62.520835876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.369004338979721,
+      "epoch": 3.1219512195121952,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08670635521411896,
+      "kl": 3.454186935414327e-05,
+      "learning_rate": 8.718571202864597e-07,
+      "loss": 0.03175315260887146,
+      "num_tokens": 3154478.0,
+      "reward": 0.37598031759262085,
+      "reward_std": 0.32647329568862915,
+      "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+      "rewards/true_env_reward_fn/std": 0.32647326588630676,
       "step": 128,
-      "step_time": 4.9305356690001645
+      "step_time": 11.551069149999876
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.0,
-      "completions/max_terminated_length": 64.0,
-      "completions/mean_length": 49.125,
-      "completions/mean_terminated_length": 49.125,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0784690976142883,
-      "epoch": 1.048780487804878,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16007214784622192,
-      "kl": 1.2491957932070363e-05,
-      "learning_rate": 6.606416241260979e-07,
-      "loss": 0.006608985364437103,
-      "num_tokens": 531862.0,
-      "reward": 0.2934249937534332,
-      "reward_std": 0.2395382523536682,
-      "rewards/true_env_reward_fn/mean": 0.2934249937534332,
-      "rewards/true_env_reward_fn/std": 0.23953823745250702,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 68.27083587646484,
+      "completions/mean_terminated_length": 68.27083587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2507834732532501,
+      "epoch": 3.1463414634146343,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07138162106275558,
+      "kl": 3.6777758396056015e-05,
+      "learning_rate": 8.689979496279746e-07,
+      "loss": 0.001895703375339508,
+      "num_tokens": 3182339.0,
+      "reward": 0.3563499450683594,
+      "reward_std": 0.2783089876174927,
+      "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+      "rewards/true_env_reward_fn/std": 0.2783089876174927,
       "step": 129,
-      "step_time": 3.173622508000335
+      "step_time": 10.723005456000465
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 65.875,
-      "completions/mean_terminated_length": 65.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.308219850063324,
-      "epoch": 1.056910569105691,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.985446427483112e-05,
-      "kl": 1.2420873190421844e-05,
-      "learning_rate": 6.577588121859508e-07,
-      "loss": 6.241918413252279e-07,
-      "num_tokens": 535957.0,
-      "reward": 0.4817493259906769,
-      "reward_std": 0.029202036559581757,
-      "rewards/true_env_reward_fn/mean": 0.4817493259906769,
-      "rewards/true_env_reward_fn/std": 0.029202038422226906,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 71.3125,
+      "completions/mean_terminated_length": 71.3125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3551637530326843,
+      "epoch": 3.1707317073170733,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0931132510304451,
+      "kl": 6.375309385475703e-05,
+      "learning_rate": 8.66112032384275e-07,
+      "loss": -0.04869828745722771,
+      "num_tokens": 3211594.0,
+      "reward": 0.35241150856018066,
+      "reward_std": 0.2379828542470932,
+      "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+      "rewards/true_env_reward_fn/std": 0.2379828542470932,
       "step": 130,
-      "step_time": 4.251137947001553
+      "step_time": 15.15810051499966
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 67.0,
-      "completions/mean_terminated_length": 67.0,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.0767641067504883,
-      "epoch": 1.065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15786457061767578,
-      "kl": 1.8847958926926367e-05,
-      "learning_rate": 6.548529203491875e-07,
-      "loss": -0.0026272237300872803,
-      "num_tokens": 539269.0,
-      "reward": 0.536803662776947,
-      "reward_std": 0.30375123023986816,
-      "rewards/true_env_reward_fn/mean": 0.536803662776947,
-      "rewards/true_env_reward_fn/std": 0.30375123023986816,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 66.6875,
+      "completions/mean_terminated_length": 66.6875,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.212640792131424,
+      "epoch": 3.1951219512195124,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0767395943403244,
+      "kl": 4.670183352573076e-05,
+      "learning_rate": 8.631995777392644e-07,
+      "loss": -0.02368815243244171,
+      "num_tokens": 3242883.0,
+      "reward": 0.3383604884147644,
+      "reward_std": 0.31325310468673706,
+      "rewards/true_env_reward_fn/mean": 0.338360458612442,
+      "rewards/true_env_reward_fn/std": 0.31325310468673706,
       "step": 131,
-      "step_time": 3.7980547870010923
+      "step_time": 20.109428818001106
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 64.25,
-      "completions/mean_terminated_length": 64.25,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.3295028805732727,
-      "epoch": 1.0731707317073171,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1881481558084488,
-      "kl": 2.0969039724150207e-05,
-      "learning_rate": 6.519242088113085e-07,
-      "loss": 0.08431969583034515,
-      "num_tokens": 545691.0,
-      "reward": 0.24590599536895752,
-      "reward_std": 0.2047487199306488,
-      "rewards/true_env_reward_fn/mean": 0.24590599536895752,
-      "rewards/true_env_reward_fn/std": 0.2047487199306488,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 59.520835876464844,
+      "completions/mean_terminated_length": 59.520835876464844,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.3366018533706665,
+      "epoch": 3.2195121951219514,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.11203660070896149,
+      "kl": 6.134294108051108e-05,
+      "learning_rate": 8.602607968003934e-07,
+      "loss": -0.03865987807512283,
+      "num_tokens": 3268372.0,
+      "reward": 0.440601110458374,
+      "reward_std": 0.336189866065979,
+      "rewards/true_env_reward_fn/mean": 0.440601110458374,
+      "rewards/true_env_reward_fn/std": 0.336189866065979,
       "step": 132,
-      "step_time": 4.361092664001262
+      "step_time": 10.12403799699996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 57.5,
-      "completions/mean_terminated_length": 57.5,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2131375670433044,
-      "epoch": 1.08130081300813,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13930389285087585,
-      "kl": 1.1046585314034019e-05,
-      "learning_rate": 6.489729398111058e-07,
-      "loss": -0.03801802545785904,
-      "num_tokens": 550295.0,
-      "reward": 0.3215479254722595,
-      "reward_std": 0.1736886352300644,
-      "rewards/true_env_reward_fn/mean": 0.3215479254722595,
-      "rewards/true_env_reward_fn/std": 0.17368865013122559,
+      "completions/max_length": 138.0,
+      "completions/max_terminated_length": 138.0,
+      "completions/mean_length": 61.41666793823242,
+      "completions/mean_terminated_length": 61.41666793823242,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2847907543182373,
+      "epoch": 3.2439024390243905,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10071831941604614,
+      "kl": 6.808681609982159e-05,
+      "learning_rate": 8.572959025833573e-07,
+      "loss": 0.0024422593414783478,
+      "num_tokens": 3291888.0,
+      "reward": 0.3618060350418091,
+      "reward_std": 0.26743030548095703,
+      "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+      "rewards/true_env_reward_fn/std": 0.26743027567863464,
       "step": 133,
-      "step_time": 3.372364626999115
+      "step_time": 10.396350653999434
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 63.375,
-      "completions/mean_terminated_length": 63.375,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.2786019444465637,
-      "epoch": 1.089430894308943,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12932609021663666,
-      "kl": 1.340499647994875e-05,
-      "learning_rate": 6.459993776071815e-07,
-      "loss": 0.029022663831710815,
-      "num_tokens": 553826.0,
-      "reward": 0.4830188751220703,
-      "reward_std": 0.29014864563941956,
-      "rewards/true_env_reward_fn/mean": 0.4830188751220703,
-      "rewards/true_env_reward_fn/std": 0.29014864563941956,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 67.4375,
+      "completions/mean_terminated_length": 67.4375,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.1504567563533783,
+      "epoch": 3.2682926829268295,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.060020897537469864,
+      "kl": 3.462390031927498e-05,
+      "learning_rate": 8.543051099966557e-07,
+      "loss": 0.04882139340043068,
+      "num_tokens": 3317125.0,
+      "reward": 0.5031180381774902,
+      "reward_std": 0.2628377676010132,
+      "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+      "rewards/true_env_reward_fn/std": 0.2628377676010132,
       "step": 134,
-      "step_time": 3.215292060998763
+      "step_time": 10.961974539000039
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 49.375,
-      "completions/mean_terminated_length": 49.375,
-      "completions/min_length": 28.0,
-      "completions/min_terminated_length": 28.0,
-      "entropy": 0.9003906548023224,
-      "epoch": 1.0975609756097562,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15958240628242493,
-      "kl": 2.838099044311093e-05,
-      "learning_rate": 6.430037884542861e-07,
-      "loss": 0.11459673941135406,
-      "num_tokens": 557217.0,
-      "reward": 0.494448184967041,
-      "reward_std": 0.3076546788215637,
-      "rewards/true_env_reward_fn/mean": 0.494448184967041,
-      "rewards/true_env_reward_fn/std": 0.3076546788215637,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 65.45833587646484,
+      "completions/mean_terminated_length": 65.45833587646484,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "entropy": 1.3224314153194427,
+      "epoch": 3.292682926829268,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10321197658777237,
+      "kl": 6.988596032897476e-05,
+      "learning_rate": 8.51288635826016e-07,
+      "loss": 0.011862488463521004,
+      "num_tokens": 3347059.0,
+      "reward": 0.39905214309692383,
+      "reward_std": 0.31803515553474426,
+      "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+      "rewards/true_env_reward_fn/std": 0.31803515553474426,
       "step": 135,
-      "step_time": 3.500462582000182
+      "step_time": 11.779171687000144
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 93.0,
       "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 57.5,
-      "completions/mean_terminated_length": 57.5,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2928712964057922,
-      "epoch": 1.1056910569105691,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23077522218227386,
-      "kl": 2.251418845844455e-05,
-      "learning_rate": 6.399864405794782e-07,
-      "loss": -0.05874824523925781,
-      "num_tokens": 562421.0,
-      "reward": 0.2385583370923996,
-      "reward_std": 0.23380905389785767,
-      "rewards/true_env_reward_fn/mean": 0.2385583370923996,
-      "rewards/true_env_reward_fn/std": 0.23380906879901886,
+      "completions/mean_length": 60.458335876464844,
+      "completions/mean_terminated_length": 60.458335876464844,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.1519232094287872,
+      "epoch": 3.317073170731707,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.085839182138443,
+      "kl": 5.779342154710321e-05,
+      "learning_rate": 8.482466987186785e-07,
+      "loss": 0.05022352561354637,
+      "num_tokens": 3370225.0,
+      "reward": 0.4742569923400879,
+      "reward_std": 0.3171122074127197,
+      "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+      "rewards/true_env_reward_fn/std": 0.3171122074127197,
       "step": 136,
-      "step_time": 4.208805245998519
+      "step_time": 8.779588141000204
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 63.25,
-      "completions/mean_terminated_length": 63.25,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 0.8736326098442078,
-      "epoch": 1.113821138211382,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.871674071997404e-05,
-      "kl": 1.1485328741400735e-05,
-      "learning_rate": 6.369476041581066e-07,
-      "loss": 5.747077125306532e-07,
-      "num_tokens": 566387.0,
-      "reward": 0.4902166724205017,
-      "reward_std": 0.038254011422395706,
-      "rewards/true_env_reward_fn/mean": 0.4902166724205017,
-      "rewards/true_env_reward_fn/std": 0.038254011422395706,
+      "completions/max_length": 130.0,
+      "completions/max_terminated_length": 130.0,
+      "completions/mean_length": 65.6875,
+      "completions/mean_terminated_length": 65.6875,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.377644658088684,
+      "epoch": 3.341463414634146,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05921673774719238,
+      "kl": 3.667381497507449e-05,
+      "learning_rate": 8.451795191675487e-07,
+      "loss": 0.020366013050079346,
+      "num_tokens": 3399578.0,
+      "reward": 0.3279460370540619,
+      "reward_std": 0.4147447645664215,
+      "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+      "rewards/true_env_reward_fn/std": 0.4147447645664215,
       "step": 137,
-      "step_time": 3.981489739000608
+      "step_time": 11.74765996799988
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 60.25,
-      "completions/mean_terminated_length": 60.25,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.1538971662521362,
-      "epoch": 1.1219512195121952,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14676779508590698,
-      "kl": 1.1651037766569061e-05,
-      "learning_rate": 6.338875512896188e-07,
-      "loss": 0.1347643882036209,
-      "num_tokens": 569341.0,
-      "reward": 0.43844783306121826,
-      "reward_std": 0.16067014634609222,
-      "rewards/true_env_reward_fn/mean": 0.43844783306121826,
-      "rewards/true_env_reward_fn/std": 0.16067016124725342,
+      "completions/max_length": 112.0,
+      "completions/max_terminated_length": 112.0,
+      "completions/mean_length": 61.583335876464844,
+      "completions/mean_terminated_length": 61.583335876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2057753205299377,
+      "epoch": 3.3658536585365852,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08148445934057236,
+      "kl": 3.0601177968492266e-05,
+      "learning_rate": 8.420873194952152e-07,
+      "loss": 0.005453992635011673,
+      "num_tokens": 3417734.0,
+      "reward": 0.5946073532104492,
+      "reward_std": 0.25090643763542175,
+      "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+      "rewards/true_env_reward_fn/std": 0.25090643763542175,
       "step": 138,
-      "step_time": 3.689221037999232
+      "step_time": 8.8135579650002
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 54.625,
-      "completions/mean_terminated_length": 54.625,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.1961707472801208,
-      "epoch": 1.1300813008130082,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16650564968585968,
-      "kl": 1.4349476259667426e-05,
-      "learning_rate": 6.308065559731976e-07,
-      "loss": 0.007910434156656265,
-      "num_tokens": 574046.0,
-      "reward": 0.4596000015735626,
-      "reward_std": 0.07715634256601334,
-      "rewards/true_env_reward_fn/mean": 0.4596000015735626,
-      "rewards/true_env_reward_fn/std": 0.07715633511543274,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 66.47917175292969,
+      "completions/mean_terminated_length": 66.47917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3243012130260468,
+      "epoch": 3.3902439024390243,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06626639515161514,
+      "kl": 3.7586045436910354e-05,
+      "learning_rate": 8.389703238378338e-07,
+      "loss": -0.003325441852211952,
+      "num_tokens": 3441749.0,
+      "reward": 0.48056626319885254,
+      "reward_std": 0.2497076690196991,
+      "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+      "rewards/true_env_reward_fn/std": 0.2497076541185379,
       "step": 139,
-      "step_time": 3.6711935700004688
+      "step_time": 9.997661417000472
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2013322114944458,
-      "epoch": 1.1382113821138211,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14415834844112396,
-      "kl": 1.4664098671346437e-05,
-      "learning_rate": 6.277048940832264e-07,
-      "loss": -0.016162052750587463,
-      "num_tokens": 576769.0,
-      "reward": 0.6152583360671997,
-      "reward_std": 0.07727260142564774,
-      "rewards/true_env_reward_fn/mean": 0.6152583360671997,
-      "rewards/true_env_reward_fn/std": 0.07727260142564774,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 189.0,
+      "completions/mean_length": 75.79167175292969,
+      "completions/mean_terminated_length": 75.79167175292969,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2918364107608795,
+      "epoch": 3.4146341463414633,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0600166842341423,
+      "kl": 3.408677366678603e-05,
+      "learning_rate": 8.358287581288822e-07,
+      "loss": -0.002709554508328438,
+      "num_tokens": 3473139.0,
+      "reward": 0.38171443343162537,
+      "reward_std": 0.2058144509792328,
+      "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+      "rewards/true_env_reward_fn/std": 0.2058144509792328,
       "step": 140,
-      "step_time": 3.5191362610003125
+      "step_time": 14.679971276999822
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 52.0,
-      "completions/max_terminated_length": 52.0,
-      "completions/mean_length": 44.75,
-      "completions/mean_terminated_length": 44.75,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.0287770330905914,
-      "epoch": 1.146341463414634,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.892051457427442e-05,
-      "kl": 1.1797974821092794e-05,
-      "learning_rate": 6.245828433445872e-07,
-      "loss": 5.92092192164273e-07,
-      "num_tokens": 578843.0,
-      "reward": 0.6387845277786255,
-      "reward_std": 0.13867565989494324,
-      "rewards/true_env_reward_fn/mean": 0.6387845277786255,
-      "rewards/true_env_reward_fn/std": 0.13867565989494324,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 62.6875,
+      "completions/mean_terminated_length": 62.6875,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.32420814037323,
+      "epoch": 3.4390243902439024,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08056586235761642,
+      "kl": 4.234552670823177e-05,
+      "learning_rate": 8.326628500827825e-07,
+      "loss": 0.019535928964614868,
+      "num_tokens": 3499324.0,
+      "reward": 0.4587298631668091,
+      "reward_std": 0.3119663596153259,
+      "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+      "rewards/true_env_reward_fn/std": 0.3119663596153259,
       "step": 141,
-      "step_time": 2.5015027329991426
+      "step_time": 11.64747691499997
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 50.75,
-      "completions/mean_terminated_length": 50.75,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 69.27083587646484,
+      "completions/mean_terminated_length": 69.27083587646484,
       "completions/min_length": 37.0,
       "completions/min_terminated_length": 37.0,
-      "entropy": 1.1590029001235962,
-      "epoch": 1.1544715447154472,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.160966694355011,
-      "kl": 1.4735675904375967e-05,
-      "learning_rate": 6.214406833077937e-07,
-      "loss": 0.0170527845621109,
-      "num_tokens": 583201.0,
-      "reward": 0.36017733812332153,
-      "reward_std": 0.3556094467639923,
-      "rewards/true_env_reward_fn/mean": 0.36017733812332153,
-      "rewards/true_env_reward_fn/std": 0.3556094467639923,
+      "entropy": 1.3127666413784027,
+      "epoch": 3.4634146341463414,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07890015095472336,
+      "kl": 4.281656902094255e-05,
+      "learning_rate": 8.294728291783965e-07,
+      "loss": -0.034988921135663986,
+      "num_tokens": 3516425.0,
+      "reward": 0.6331583261489868,
+      "reward_std": 0.2317410111427307,
+      "rewards/true_env_reward_fn/mean": 0.633158266544342,
+      "rewards/true_env_reward_fn/std": 0.23174098134040833,
       "step": 142,
-      "step_time": 3.2783409929998015
+      "step_time": 8.380270293999729
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 67.0,
-      "completions/mean_terminated_length": 67.0,
-      "completions/min_length": 59.0,
-      "completions/min_terminated_length": 59.0,
-      "entropy": 1.1985241174697876,
-      "epoch": 1.1626016260162602,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12272457778453827,
-      "kl": 1.7849098185251933e-05,
-      "learning_rate": 6.182786953239593e-07,
-      "loss": -0.0016125142574310303,
-      "num_tokens": 587317.0,
-      "reward": 0.34745320677757263,
-      "reward_std": 0.3954337239265442,
-      "rewards/true_env_reward_fn/mean": 0.34745320677757263,
-      "rewards/true_env_reward_fn/std": 0.3954337537288666,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 73.97917175292969,
+      "completions/mean_terminated_length": 73.97917175292969,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.323029786348343,
+      "epoch": 3.4878048780487805,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08455090969800949,
+      "kl": 6.877856139908545e-05,
+      "learning_rate": 8.262589266423908e-07,
+      "loss": 0.06993371993303299,
+      "num_tokens": 3542912.0,
+      "reward": 0.41727983951568604,
+      "reward_std": 0.23754946887493134,
+      "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+      "rewards/true_env_reward_fn/std": 0.23754946887493134,
       "step": 143,
-      "step_time": 3.9932043310000154
+      "step_time": 11.716556537000088
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 48.0,
-      "completions/max_terminated_length": 48.0,
-      "completions/mean_length": 43.75,
-      "completions/mean_terminated_length": 43.75,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1116944551467896,
-      "epoch": 1.170731707317073,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18435250222682953,
-      "kl": 1.014559029499651e-05,
-      "learning_rate": 6.150971625196048e-07,
-      "loss": 0.009793907403945923,
-      "num_tokens": 590191.0,
-      "reward": 0.4938516616821289,
-      "reward_std": 0.03703190013766289,
-      "rewards/true_env_reward_fn/mean": 0.4938516616821289,
-      "rewards/true_env_reward_fn/std": 0.037031903862953186,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 63.3125,
+      "completions/mean_terminated_length": 63.3125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3465435802936554,
+      "epoch": 3.5121951219512195,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09822116792201996,
+      "kl": 8.00468278612243e-05,
+      "learning_rate": 8.230213754324772e-07,
+      "loss": 0.07691670209169388,
+      "num_tokens": 3569575.0,
+      "reward": 0.28445714712142944,
+      "reward_std": 0.33810389041900635,
+      "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+      "rewards/true_env_reward_fn/std": 0.33810392022132874,
       "step": 144,
-      "step_time": 2.3663663690022076
+      "step_time": 10.67718802499985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.0,
-      "completions/mean_terminated_length": 53.0,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.413200855255127,
-      "epoch": 1.1788617886178863,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14336225390434265,
-      "kl": 2.0541991034406237e-05,
-      "learning_rate": 6.118963697713078e-07,
-      "loss": -0.013927727937698364,
-      "num_tokens": 593671.0,
-      "reward": 0.4619143605232239,
-      "reward_std": 0.3773181140422821,
-      "rewards/true_env_reward_fn/mean": 0.4619143605232239,
-      "rewards/true_env_reward_fn/std": 0.3773181140422821,
+      "completions/max_length": 126.0,
+      "completions/max_terminated_length": 126.0,
+      "completions/mean_length": 74.375,
+      "completions/mean_terminated_length": 74.375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3208706378936768,
+      "epoch": 3.5365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06776741147041321,
+      "kl": 4.5862597744417144e-05,
+      "learning_rate": 8.19760410220527e-07,
+      "loss": -0.014808554202318192,
+      "num_tokens": 3589641.0,
+      "reward": 0.5829761028289795,
+      "reward_std": 0.21224236488342285,
+      "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+      "rewards/true_env_reward_fn/std": 0.21224237978458405,
       "step": 145,
-      "step_time": 3.9730388410007436
+      "step_time": 8.610043666999445
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 128.0,
-      "completions/max_terminated_length": 128.0,
-      "completions/mean_length": 68.625,
-      "completions/mean_terminated_length": 68.625,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "entropy": 1.08676016330719,
-      "epoch": 1.1869918699186992,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15555191040039062,
-      "kl": 1.6947700260061538e-05,
-      "learning_rate": 6.086766036801937e-07,
-      "loss": -0.139797180891037,
-      "num_tokens": 601612.0,
-      "reward": 0.3831036686897278,
-      "reward_std": 0.09242849797010422,
-      "rewards/true_env_reward_fn/mean": 0.3831036686897278,
-      "rewards/true_env_reward_fn/std": 0.09242849797010422,
+      "completions/max_length": 180.0,
+      "completions/max_terminated_length": 180.0,
+      "completions/mean_length": 72.1875,
+      "completions/mean_terminated_length": 72.1875,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2934723794460297,
+      "epoch": 3.5609756097560976,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.05893269553780556,
+      "kl": 3.648432630143361e-05,
+      "learning_rate": 8.164762673755609e-07,
+      "loss": 0.023374930024147034,
+      "num_tokens": 3615570.0,
+      "reward": 0.47375163435935974,
+      "reward_std": 0.16054874658584595,
+      "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+      "rewards/true_env_reward_fn/std": 0.16054873168468475,
       "step": 146,
-      "step_time": 6.323679949000507
+      "step_time": 13.649344002000362
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 67.5,
-      "completions/mean_terminated_length": 67.5,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.5055813789367676,
-      "epoch": 1.1951219512195121,
+      "completions/max_length": 125.0,
+      "completions/max_terminated_length": 125.0,
+      "completions/mean_length": 72.1875,
+      "completions/mean_terminated_length": 72.1875,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3180726766586304,
+      "epoch": 3.5853658536585367,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2397669553756714,
-      "kl": 2.704876442294335e-05,
-      "learning_rate": 6.054381525462745e-07,
-      "loss": 0.2738838493824005,
-      "num_tokens": 606712.0,
-      "reward": 0.25339600443840027,
-      "reward_std": 0.3023079037666321,
-      "rewards/true_env_reward_fn/mean": 0.25339600443840027,
-      "rewards/true_env_reward_fn/std": 0.3023079037666321,
+      "grad_norm": 0.08518138527870178,
+      "kl": 6.788871905882843e-05,
+      "learning_rate": 8.131691849466152e-07,
+      "loss": -0.04987313598394394,
+      "num_tokens": 3637475.0,
+      "reward": 0.5195532441139221,
+      "reward_std": 0.26043611764907837,
+      "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+      "rewards/true_env_reward_fn/std": 0.26043611764907837,
       "step": 147,
-      "step_time": 5.185072233998653
+      "step_time": 11.702765863000877
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 47.25,
-      "completions/mean_terminated_length": 47.25,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.135968267917633,
-      "epoch": 1.203252032520325,
+      "completions/max_length": 168.0,
+      "completions/max_terminated_length": 168.0,
+      "completions/mean_length": 73.83333587646484,
+      "completions/mean_terminated_length": 73.83333587646484,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "entropy": 1.167496383190155,
+      "epoch": 3.6097560975609757,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2614514231681824,
-      "kl": 3.613240005506668e-05,
-      "learning_rate": 6.021813063426323e-07,
-      "loss": 0.10286401212215424,
-      "num_tokens": 610566.0,
-      "reward": 0.31031692028045654,
-      "reward_std": 0.3124054968357086,
-      "rewards/true_env_reward_fn/mean": 0.31031692028045654,
-      "rewards/true_env_reward_fn/std": 0.312405526638031,
+      "grad_norm": 0.07978484779596329,
+      "kl": 6.430712710425723e-05,
+      "learning_rate": 8.098394026454884e-07,
+      "loss": 0.024383332580327988,
+      "num_tokens": 3663171.0,
+      "reward": 0.4524516761302948,
+      "reward_std": 0.2587544322013855,
+      "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+      "rewards/true_env_reward_fn/std": 0.2587544322013855,
       "step": 148,
-      "step_time": 3.2177847610000754
+      "step_time": 13.306644664000487
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 52.75,
-      "completions/mean_terminated_length": 52.75,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 1.4589928984642029,
-      "epoch": 1.2113821138211383,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2019941508769989,
-      "kl": 2.1841721718374174e-05,
-      "learning_rate": 5.989063566894572e-07,
-      "loss": 0.010915875434875488,
-      "num_tokens": 615716.0,
-      "reward": 0.31711751222610474,
-      "reward_std": 0.13289952278137207,
-      "rewards/true_env_reward_fn/mean": 0.31711751222610474,
-      "rewards/true_env_reward_fn/std": 0.13289952278137207,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 65.875,
+      "completions/mean_terminated_length": 65.875,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.155810385942459,
+      "epoch": 3.6341463414634148,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.046879056841135025,
+      "kl": 4.4023097871104255e-05,
+      "learning_rate": 8.064871618293645e-07,
+      "loss": -0.01477135717868805,
+      "num_tokens": 3683813.0,
+      "reward": 0.6097190380096436,
+      "reward_std": 0.17910261452198029,
+      "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+      "rewards/true_env_reward_fn/std": 0.17910261452198029,
       "step": 149,
-      "step_time": 4.3804878079990885
+      "step_time": 9.446422488999815
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4188,26 +4188,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 67.375,
-      "completions/mean_terminated_length": 67.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.1892729997634888,
-      "epoch": 1.2195121951219512,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.835455471649766e-05,
-      "kl": 1.3420096820482286e-05,
-      "learning_rate": 5.956135968279332e-07,
-      "loss": 6.646802717114042e-07,
-      "num_tokens": 619439.0,
-      "reward": 0.6557307243347168,
-      "reward_std": 0.2151959389448166,
-      "rewards/true_env_reward_fn/mean": 0.6557307243347168,
-      "rewards/true_env_reward_fn/std": 0.21519595384597778,
+      "completions/max_length": 162.0,
+      "completions/max_terminated_length": 162.0,
+      "completions/mean_length": 72.8125,
+      "completions/mean_terminated_length": 72.8125,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3542745113372803,
+      "epoch": 3.658536585365854,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06843585520982742,
+      "kl": 3.291011944384081e-05,
+      "learning_rate": 8.03112705483319e-07,
+      "loss": 0.009258950129151344,
+      "num_tokens": 3702516.0,
+      "reward": 0.5345131754875183,
+      "reward_std": 0.22612926363945007,
+      "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+      "rewards/true_env_reward_fn/std": 0.22612926363945007,
       "step": 150,
-      "step_time": 3.63938895299907
+      "step_time": 10.538116119999813
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4216,26 +4216,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 68.75,
-      "completions/mean_terminated_length": 68.75,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.1402934789657593,
-      "epoch": 1.2276422764227641,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010795716661959887,
-      "kl": 1.4652535810455447e-05,
-      "learning_rate": 5.923033215939834e-07,
-      "loss": 7.542968205598299e-07,
-      "num_tokens": 621009.0,
-      "reward": 0.8541955947875977,
-      "reward_std": 0.09160846471786499,
-      "rewards/true_env_reward_fn/mean": 0.8541955947875977,
-      "rewards/true_env_reward_fn/std": 0.09160846471786499,
+      "completions/max_length": 178.0,
+      "completions/max_terminated_length": 178.0,
+      "completions/mean_length": 66.58333587646484,
+      "completions/mean_terminated_length": 66.58333587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.3427793979644775,
+      "epoch": 3.682926829268293,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0966033861041069,
+      "kl": 7.367974285443779e-05,
+      "learning_rate": 7.99716278202706e-07,
+      "loss": 0.07012784481048584,
+      "num_tokens": 3733800.0,
+      "reward": 0.3090733289718628,
+      "reward_std": 0.3846965730190277,
+      "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+      "rewards/true_env_reward_fn/std": 0.3846965730190277,
       "step": 151,
-      "step_time": 3.926544339999964
+      "step_time": 15.755764130999978
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4244,26 +4244,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
       "completions/mean_length": 64.125,
       "completions/mean_terminated_length": 64.125,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.190350890159607,
-      "epoch": 1.2357723577235773,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19853363931179047,
-      "kl": 2.269768037876929e-05,
-      "learning_rate": 5.889758273918683e-07,
-      "loss": 0.044217392802238464,
-      "num_tokens": 623994.0,
-      "reward": 0.4411996603012085,
-      "reward_std": 0.2517909109592438,
-      "rewards/true_env_reward_fn/mean": 0.4411996603012085,
-      "rewards/true_env_reward_fn/std": 0.2517908811569214,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3039455115795135,
+      "epoch": 3.7073170731707314,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06908538937568665,
+      "kl": 4.5496621623897227e-05,
+      "learning_rate": 7.962981261754294e-07,
+      "loss": 0.02471787855029106,
+      "num_tokens": 3758710.0,
+      "reward": 0.35497602820396423,
+      "reward_std": 0.25890877842903137,
+      "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+      "rewards/true_env_reward_fn/std": 0.25890880823135376,
       "step": 152,
-      "step_time": 3.7339736520007136
+      "step_time": 9.670861957999932
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4272,26 +4272,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 73.75,
-      "completions/mean_terminated_length": 73.75,
-      "completions/min_length": 56.0,
-      "completions/min_terminated_length": 56.0,
-      "entropy": 1.2316884994506836,
-      "epoch": 1.2439024390243902,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.35867181699723e-05,
-      "kl": 1.1579370038816705e-05,
-      "learning_rate": 5.856314121676467e-07,
-      "loss": 5.79387460675207e-07,
-      "num_tokens": 628224.0,
-      "reward": 0.6024306416511536,
-      "reward_std": 0.13815335929393768,
-      "rewards/true_env_reward_fn/mean": 0.6024306416511536,
-      "rewards/true_env_reward_fn/std": 0.13815335929393768,
+      "completions/max_length": 369.0,
+      "completions/max_terminated_length": 369.0,
+      "completions/mean_length": 71.75,
+      "completions/mean_terminated_length": 71.75,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.342434972524643,
+      "epoch": 3.7317073170731705,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0625183954834938,
+      "kl": 5.662065541400807e-05,
+      "learning_rate": 7.928584971640974e-07,
+      "loss": 0.15159915387630463,
+      "num_tokens": 3781818.0,
+      "reward": 0.456516832113266,
+      "reward_std": 0.291423499584198,
+      "rewards/true_env_reward_fn/mean": 0.456516832113266,
+      "rewards/true_env_reward_fn/std": 0.291423499584198,
       "step": 153,
-      "step_time": 4.375236807001784
+      "step_time": 22.82054339000024
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4300,26 +4300,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 137.0,
-      "completions/max_terminated_length": 137.0,
-      "completions/mean_length": 60.375,
-      "completions/mean_terminated_length": 60.375,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 1.4028943181037903,
-      "epoch": 1.2520325203252032,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.25418931245803833,
-      "kl": 3.6890452065563295e-05,
-      "learning_rate": 5.822703753824966e-07,
-      "loss": 0.25599968433380127,
-      "num_tokens": 631183.0,
-      "reward": 0.38683533668518066,
-      "reward_std": 0.43613559007644653,
-      "rewards/true_env_reward_fn/mean": 0.38683533668518066,
-      "rewards/true_env_reward_fn/std": 0.43613559007644653,
+      "completions/max_length": 102.0,
+      "completions/max_terminated_length": 102.0,
+      "completions/mean_length": 65.0,
+      "completions/mean_terminated_length": 65.0,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.256364792585373,
+      "epoch": 3.7560975609756095,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0713125616312027,
+      "kl": 5.741999029851286e-05,
+      "learning_rate": 7.893976404880641e-07,
+      "loss": 0.0011316314339637756,
+      "num_tokens": 3801434.0,
+      "reward": 0.6220619082450867,
+      "reward_std": 0.260771244764328,
+      "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+      "rewards/true_env_reward_fn/std": 0.260771244764328,
       "step": 154,
-      "step_time": 6.055355972999678
+      "step_time": 9.973958625999785
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4328,26 +4328,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 68.875,
-      "completions/mean_terminated_length": 68.875,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.367663562297821,
-      "epoch": 1.2601626016260163,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13331811130046844,
-      "kl": 1.2863993106293492e-05,
-      "learning_rate": 5.788930179859024e-07,
-      "loss": -0.0047044456005096436,
-      "num_tokens": 636230.0,
-      "reward": 0.3489508628845215,
-      "reward_std": 0.13627417385578156,
-      "rewards/true_env_reward_fn/mean": 0.3489508628845215,
-      "rewards/true_env_reward_fn/std": 0.13627417385578156,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 75.02083587646484,
+      "completions/mean_terminated_length": 75.02083587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3467005491256714,
+      "epoch": 3.7804878048780486,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05885510891675949,
+      "kl": 5.8398384226165945e-05,
+      "learning_rate": 7.859158070053576e-07,
+      "loss": -0.006662093102931976,
+      "num_tokens": 3829875.0,
+      "reward": 0.4248020052909851,
+      "reward_std": 0.17955487966537476,
+      "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+      "rewards/true_env_reward_fn/std": 0.17955489456653595,
       "step": 155,
-      "step_time": 3.9174396130001696
+      "step_time": 9.758407419000378
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4356,26 +4356,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 135.0,
-      "completions/max_terminated_length": 135.0,
-      "completions/mean_length": 68.5,
-      "completions/mean_terminated_length": 68.5,
+      "completions/max_length": 119.0,
+      "completions/max_terminated_length": 119.0,
+      "completions/mean_length": 67.22917175292969,
+      "completions/mean_terminated_length": 67.22917175292969,
       "completions/min_length": 41.0,
       "completions/min_terminated_length": 41.0,
-      "entropy": 1.2759611010551453,
-      "epoch": 1.2682926829268293,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12738144397735596,
-      "kl": 1.5844128029129934e-05,
-      "learning_rate": 5.754996423887061e-07,
-      "loss": -0.011055335402488708,
-      "num_tokens": 640262.0,
-      "reward": 0.344404935836792,
-      "reward_std": 0.16542991995811462,
-      "rewards/true_env_reward_fn/mean": 0.344404935836792,
-      "rewards/true_env_reward_fn/std": 0.16542991995811462,
+      "entropy": 1.2418483197689056,
+      "epoch": 3.8048780487804876,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07287121564149857,
+      "kl": 7.330268545047147e-05,
+      "learning_rate": 7.824132490944967e-07,
+      "loss": 0.009832290932536125,
+      "num_tokens": 3858478.0,
+      "reward": 0.45385628938674927,
+      "reward_std": 0.241779163479805,
+      "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+      "rewards/true_env_reward_fn/std": 0.2417791783809662,
       "step": 156,
-      "step_time": 5.706334413998775
+      "step_time": 11.009583763999672
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4384,26 +4384,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 59.625,
-      "completions/mean_terminated_length": 59.625,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.0904476642608643,
-      "epoch": 1.2764227642276422,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.733699021628127e-05,
-      "kl": 1.2238857834745431e-05,
-      "learning_rate": 5.720905524360308e-07,
-      "loss": 6.076299996493617e-07,
-      "num_tokens": 645091.0,
-      "reward": 0.4731999933719635,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.4731999933719635,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 62.25,
+      "completions/mean_terminated_length": 62.25,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2394451797008514,
+      "epoch": 3.8292682926829267,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07873199880123138,
+      "kl": 3.887376169586787e-05,
+      "learning_rate": 7.788902206361973e-07,
+      "loss": 0.004030962474644184,
+      "num_tokens": 3882682.0,
+      "reward": 0.5042052268981934,
+      "reward_std": 0.17870797216892242,
+      "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+      "rewards/true_env_reward_fn/std": 0.17870797216892242,
       "step": 157,
-      "step_time": 3.7696847109982627
+      "step_time": 10.12789283499933
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4412,26 +4412,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 60.0,
-      "completions/mean_terminated_length": 60.0,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.3856677412986755,
-      "epoch": 1.2845528455284554,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18670028448104858,
-      "kl": 1.8415606064081658e-05,
-      "learning_rate": 5.686660533800736e-07,
-      "loss": -0.07078710198402405,
-      "num_tokens": 648179.0,
-      "reward": 0.537517786026001,
-      "reward_std": 0.1451217085123062,
-      "rewards/true_env_reward_fn/mean": 0.537517786026001,
-      "rewards/true_env_reward_fn/std": 0.1451217085123062,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 60.458335876464844,
+      "completions/mean_terminated_length": 60.458335876464844,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.148638516664505,
+      "epoch": 3.8536585365853657,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06768295168876648,
+      "kl": 5.1569048991950694e-05,
+      "learning_rate": 7.7534697699497e-07,
+      "loss": -0.018120331689715385,
+      "num_tokens": 3902176.0,
+      "reward": 0.5385247468948364,
+      "reward_std": 0.20308326184749603,
+      "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+      "rewards/true_env_reward_fn/std": 0.20308324694633484,
       "step": 158,
-      "step_time": 3.7075291149994882
+      "step_time": 7.800485663000472
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4440,26 +4440,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 68.125,
-      "completions/mean_terminated_length": 68.125,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.121916651725769,
-      "epoch": 1.2926829268292683,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11415883898735046,
-      "kl": 1.909901220642496e-05,
-      "learning_rate": 5.652264518527725e-07,
-      "loss": -0.04401372745633125,
-      "num_tokens": 652044.0,
-      "reward": 0.5182899832725525,
-      "reward_std": 0.21869486570358276,
-      "rewards/true_env_reward_fn/mean": 0.5182899832725525,
-      "rewards/true_env_reward_fn/std": 0.21869485080242157,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 67.875,
+      "completions/mean_terminated_length": 67.875,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.0718395709991455,
+      "epoch": 3.8780487804878048,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08181443065404892,
+      "kl": 6.815949382144026e-05,
+      "learning_rate": 7.717837750006106e-07,
+      "loss": -0.007854004390537739,
+      "num_tokens": 3930658.0,
+      "reward": 0.4394054412841797,
+      "reward_std": 0.3746899962425232,
+      "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+      "rewards/true_env_reward_fn/std": 0.3746899962425232,
       "step": 159,
-      "step_time": 3.8929355969994504
+      "step_time": 13.77649076500029
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4468,26 +4468,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 143.0,
-      "completions/max_terminated_length": 143.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.5205118060112,
-      "epoch": 1.3008130081300813,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.2403375506401062,
-      "kl": 3.7574073758150917e-05,
-      "learning_rate": 5.617720558383508e-07,
-      "loss": 0.26385918259620667,
-      "num_tokens": 656362.0,
-      "reward": 0.41201668977737427,
-      "reward_std": 0.2023741453886032,
-      "rewards/true_env_reward_fn/mean": 0.41201668977737427,
-      "rewards/true_env_reward_fn/std": 0.20237413048744202,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 65.33333587646484,
+      "completions/mean_terminated_length": 65.33333587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.135006695985794,
+      "epoch": 3.902439024390244,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07245675474405289,
+      "kl": 6.525267690449255e-05,
+      "learning_rate": 7.682008729295833e-07,
+      "loss": 0.07331673055887222,
+      "num_tokens": 3958082.0,
+      "reward": 0.3795818090438843,
+      "reward_std": 0.21483220160007477,
+      "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+      "rewards/true_env_reward_fn/std": 0.21483221650123596,
       "step": 160,
-      "step_time": 6.157555950998358
+      "step_time": 13.25029361300085
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4496,26 +4496,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 67.375,
-      "completions/mean_terminated_length": 67.375,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.3556928038597107,
-      "epoch": 1.3089430894308944,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18135924637317657,
-      "kl": 2.6372636057203636e-05,
-      "learning_rate": 5.583031746457407e-07,
-      "loss": -0.10538280755281448,
-      "num_tokens": 659977.0,
-      "reward": 0.4239906072616577,
-      "reward_std": 0.3287450969219208,
-      "rewards/true_env_reward_fn/mean": 0.4239906072616577,
-      "rewards/true_env_reward_fn/std": 0.3287450969219208,
+      "completions/max_length": 286.0,
+      "completions/max_terminated_length": 286.0,
+      "completions/mean_length": 80.66667175292969,
+      "completions/mean_terminated_length": 80.66667175292969,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.242073804140091,
+      "epoch": 3.926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.06932233273983002,
+      "kl": 6.277856755332323e-05,
+      "learning_rate": 7.645985304863003e-07,
+      "loss": 0.05312003195285797,
+      "num_tokens": 3984066.0,
+      "reward": 0.4469220042228699,
+      "reward_std": 0.17845165729522705,
+      "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+      "rewards/true_env_reward_fn/std": 0.17845165729522705,
       "step": 161,
-      "step_time": 4.709477423999488
+      "step_time": 17.837881629000094
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4524,26 +4524,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 71.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 52.5,
-      "completions/mean_terminated_length": 52.5,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 0.9743769466876984,
-      "epoch": 1.3170731707317074,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.178797647356987,
-      "kl": 1.2532927030406427e-05,
-      "learning_rate": 5.548201188808869e-07,
-      "loss": -0.04164513945579529,
-      "num_tokens": 661409.0,
-      "reward": 0.8041956424713135,
-      "reward_std": 0.1363772451877594,
-      "rewards/true_env_reward_fn/mean": 0.8041956424713135,
-      "rewards/true_env_reward_fn/std": 0.1363772302865982,
+      "completions/max_length": 85.0,
+      "completions/max_terminated_length": 85.0,
+      "completions/mean_length": 52.145835876464844,
+      "completions/mean_terminated_length": 52.145835876464844,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.1892985105514526,
+      "epoch": 3.951219512195122,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.10256654024124146,
+      "kl": 7.12887790541572e-05,
+      "learning_rate": 7.609770087842968e-07,
+      "loss": -0.046506255865097046,
+      "num_tokens": 4003625.0,
+      "reward": 0.49098464846611023,
+      "reward_std": 0.3060121238231659,
+      "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+      "rewards/true_env_reward_fn/std": 0.3060121238231659,
       "step": 162,
-      "step_time": 3.1791253910014348
+      "step_time": 9.286757633000889
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4552,26 +4552,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 64.875,
-      "completions/mean_terminated_length": 64.875,
-      "completions/min_length": 56.0,
-      "completions/min_terminated_length": 56.0,
-      "entropy": 1.3072250485420227,
-      "epoch": 1.3252032520325203,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16768279671669006,
-      "kl": 1.89386219062726e-05,
-      "learning_rate": 5.513232004189339e-07,
-      "loss": -0.01292814314365387,
-      "num_tokens": 666504.0,
-      "reward": 0.27981066703796387,
-      "reward_std": 0.2949208915233612,
-      "rewards/true_env_reward_fn/mean": 0.27981066703796387,
-      "rewards/true_env_reward_fn/std": 0.2949208915233612,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 63.395835876464844,
+      "completions/mean_terminated_length": 63.395835876464844,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.1780613362789154,
+      "epoch": 3.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07468484342098236,
+      "kl": 5.8644353430281626e-05,
+      "learning_rate": 7.573365703273045e-07,
+      "loss": -0.0016099847853183746,
+      "num_tokens": 4024676.0,
+      "reward": 0.5258157253265381,
+      "reward_std": 0.2065279185771942,
+      "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+      "rewards/true_env_reward_fn/std": 0.2065279185771942,
       "step": 163,
-      "step_time": 3.778431355000066
+      "step_time": 10.785562561999996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4580,26 +4580,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 66.0,
-      "completions/max_terminated_length": 66.0,
-      "completions/mean_length": 51.0,
-      "completions/mean_terminated_length": 51.0,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.1984856128692627,
-      "epoch": 1.3333333333333333,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13123764097690582,
-      "kl": 1.9091786271019373e-05,
-      "learning_rate": 5.478127323763027e-07,
-      "loss": 0.035523779690265656,
-      "num_tokens": 671004.0,
-      "reward": 0.2771500051021576,
-      "reward_std": 0.30146247148513794,
-      "rewards/true_env_reward_fn/mean": 0.2771500051021576,
-      "rewards/true_env_reward_fn/std": 0.30146247148513794,
+      "completions/max_length": 177.0,
+      "completions/max_terminated_length": 177.0,
+      "completions/mean_length": 67.20833587646484,
+      "completions/mean_terminated_length": 67.20833587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.1701751053333282,
+      "epoch": 4.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06985389441251755,
+      "kl": 3.805391224886989e-05,
+      "learning_rate": 7.536774789902245e-07,
+      "loss": 0.0351775586605072,
+      "num_tokens": 4048798.0,
+      "reward": 0.520969033241272,
+      "reward_std": 0.118266262114048,
+      "rewards/true_env_reward_fn/mean": 0.520969033241272,
+      "rewards/true_env_reward_fn/std": 0.1182662770152092,
       "step": 164,
-      "step_time": 3.304021460000513
+      "step_time": 12.722446307000155
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4608,26 +4608,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 102.0,
-      "completions/max_terminated_length": 102.0,
-      "completions/mean_length": 84.875,
-      "completions/mean_terminated_length": 84.875,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2726752758026123,
-      "epoch": 1.3414634146341464,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15594874322414398,
-      "kl": 2.5981638827943243e-05,
-      "learning_rate": 5.442890290826518e-07,
-      "loss": -0.01398652046918869,
-      "num_tokens": 677307.0,
-      "reward": 0.35573017597198486,
-      "reward_std": 0.25944042205810547,
-      "rewards/true_env_reward_fn/mean": 0.35573017597198486,
-      "rewards/true_env_reward_fn/std": 0.25944042205810547,
+      "completions/max_length": 202.0,
+      "completions/max_terminated_length": 202.0,
+      "completions/mean_length": 62.16666793823242,
+      "completions/mean_terminated_length": 62.16666793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.1948959231376648,
+      "epoch": 4.024390243902439,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07691483944654465,
+      "kl": 6.559857092724997e-05,
+      "learning_rate": 7.5e-07,
+      "loss": 0.010169023647904396,
+      "num_tokens": 4076302.0,
+      "reward": 0.3847707509994507,
+      "reward_std": 0.2532062828540802,
+      "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+      "rewards/true_env_reward_fn/std": 0.2532062828540802,
       "step": 165,
-      "step_time": 4.972808451999299
+      "step_time": 14.89655208000022
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4636,26 +4636,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 210.0,
-      "completions/max_terminated_length": 210.0,
-      "completions/mean_length": 103.625,
-      "completions/mean_terminated_length": 103.625,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.0949090719223022,
-      "epoch": 1.3495934959349594,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.07191785424947739,
-      "kl": 1.4828182884230046e-05,
-      "learning_rate": 5.407524060527332e-07,
-      "loss": -0.05063021928071976,
-      "num_tokens": 682076.0,
-      "reward": 0.4925253391265869,
-      "reward_std": 0.10813789069652557,
-      "rewards/true_env_reward_fn/mean": 0.4925253391265869,
-      "rewards/true_env_reward_fn/std": 0.10813789069652557,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 61.645835876464844,
+      "completions/mean_terminated_length": 61.645835876464844,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.176283597946167,
+      "epoch": 4.048780487804878,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08597421646118164,
+      "kl": 7.07068675183109e-05,
+      "learning_rate": 7.463043999163918e-07,
+      "loss": 0.030138514935970306,
+      "num_tokens": 4096853.0,
+      "reward": 0.5679852962493896,
+      "reward_std": 0.23158182203769684,
+      "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+      "rewards/true_env_reward_fn/std": 0.23158180713653564,
       "step": 166,
-      "step_time": 8.769379133000257
+      "step_time": 8.962532588000613
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4664,54 +4664,54 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1857684254646301,
-      "epoch": 1.3577235772357723,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.923542191041633e-05,
-      "kl": 1.0165251751459436e-05,
-      "learning_rate": 5.37203179958141e-07,
-      "loss": 5.114516170579009e-07,
-      "num_tokens": 685500.0,
-      "reward": 0.5610077381134033,
-      "reward_std": 0.316459059715271,
-      "rewards/true_env_reward_fn/mean": 0.5610077381134033,
-      "rewards/true_env_reward_fn/std": 0.3164590299129486,
+      "completions/max_length": 172.0,
+      "completions/max_terminated_length": 172.0,
+      "completions/mean_length": 69.33333587646484,
+      "completions/mean_terminated_length": 69.33333587646484,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.3250808417797089,
+      "epoch": 4.073170731707317,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08058538287878036,
+      "kl": 6.5122869273182e-05,
+      "learning_rate": 7.425909466126568e-07,
+      "loss": 0.03569657728075981,
+      "num_tokens": 4118253.0,
+      "reward": 0.5882628560066223,
+      "reward_std": 0.2646455764770508,
+      "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+      "rewards/true_env_reward_fn/std": 0.2646455764770508,
       "step": 167,
-      "step_time": 3.1906087530005607
+      "step_time": 11.708963983000103
     },
     {
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 211.0,
-      "completions/max_terminated_length": 211.0,
-      "completions/mean_length": 74.25,
-      "completions/mean_terminated_length": 74.25,
-      "completions/min_length": 16.0,
-      "completions/min_terminated_length": 16.0,
-      "entropy": 1.527149498462677,
-      "epoch": 1.3658536585365852,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13164250552654266,
-      "kl": 1.8541333702160046e-05,
-      "learning_rate": 5.33641668598956e-07,
-      "loss": -0.2347300797700882,
-      "num_tokens": 688318.0,
-      "reward": 0.7218117713928223,
-      "reward_std": 0.1818692982196808,
-      "rewards/true_env_reward_fn/mean": 0.7218117713928223,
-      "rewards/true_env_reward_fn/std": 0.1818692833185196,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 261.0,
+      "completions/max_terminated_length": 261.0,
+      "completions/mean_length": 75.625,
+      "completions/mean_terminated_length": 75.625,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.319758415222168,
+      "epoch": 4.097560975609756,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07505878806114197,
+      "kl": 4.8000228161981795e-05,
+      "learning_rate": 7.388599092561314e-07,
+      "loss": 0.10015778243541718,
+      "num_tokens": 4142931.0,
+      "reward": 0.5041360259056091,
+      "reward_std": 0.29254013299942017,
+      "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+      "rewards/true_env_reward_fn/std": 0.2925401031970978,
       "step": 168,
-      "step_time": 8.50137474999974
+      "step_time": 16.905241970000134
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4720,26 +4720,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 76.375,
-      "completions/mean_terminated_length": 76.375,
-      "completions/min_length": 63.0,
-      "completions/min_terminated_length": 63.0,
-      "entropy": 1.3749513030052185,
-      "epoch": 1.3739837398373984,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13246971368789673,
-      "kl": 1.2620409506780561e-05,
-      "learning_rate": 5.300681908752895e-07,
-      "loss": 0.024534843862056732,
-      "num_tokens": 692541.0,
-      "reward": 0.4925253391265869,
-      "reward_std": 0.10813789069652557,
-      "rewards/true_env_reward_fn/mean": 0.4925253391265869,
-      "rewards/true_env_reward_fn/std": 0.10813789069652557,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 60.333335876464844,
+      "completions/mean_terminated_length": 60.333335876464844,
+      "completions/min_length": 17.0,
+      "completions/min_terminated_length": 17.0,
+      "entropy": 1.3414625525474548,
+      "epoch": 4.121951219512195,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08060909807682037,
+      "kl": 6.745914834027644e-05,
+      "learning_rate": 7.351115582887211e-07,
+      "loss": -0.04827923700213432,
+      "num_tokens": 4172755.0,
+      "reward": 0.4034843444824219,
+      "reward_std": 0.29997992515563965,
+      "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+      "rewards/true_env_reward_fn/std": 0.29997992515563965,
       "step": 169,
-      "step_time": 3.9512340759993094
+      "step_time": 12.27045077799994
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4747,27 +4747,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 67.875,
-      "completions/mean_terminated_length": 67.875,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2032299041748047,
-      "epoch": 1.3821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11185794323682785,
-      "kl": 1.5517784049734473e-05,
-      "learning_rate": 5.264830667587295e-07,
-      "loss": -0.05245225131511688,
-      "num_tokens": 698064.0,
-      "reward": 0.4444866180419922,
-      "reward_std": 0.32400256395339966,
-      "rewards/true_env_reward_fn/mean": 0.4444866180419922,
-      "rewards/true_env_reward_fn/std": 0.32400253415107727,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 155.0,
+      "completions/mean_length": 72.875,
+      "completions/mean_terminated_length": 63.53191375732422,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.4034385085105896,
+      "epoch": 4.146341463414634,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05382552370429039,
+      "kl": 3.640815248218132e-05,
+      "learning_rate": 7.313461654072973e-07,
+      "loss": 0.07585563510656357,
+      "num_tokens": 4196629.0,
+      "reward": 0.5297740697860718,
+      "reward_std": 0.25283464789390564,
+      "rewards/true_env_reward_fn/mean": 0.529774010181427,
+      "rewards/true_env_reward_fn/std": 0.25283464789390564,
       "step": 170,
-      "step_time": 4.471538110999973
+      "step_time": 31.721865252000498
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4776,26 +4776,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 108.0,
-      "completions/max_terminated_length": 108.0,
-      "completions/mean_length": 95.0,
-      "completions/mean_terminated_length": 95.0,
-      "completions/min_length": 79.0,
-      "completions/min_terminated_length": 79.0,
-      "entropy": 1.4204387068748474,
-      "epoch": 1.3902439024390243,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1646534949541092,
-      "kl": 2.4697198568901513e-05,
-      "learning_rate": 5.228866172636899e-07,
-      "loss": 0.02632315456867218,
-      "num_tokens": 704196.0,
-      "reward": 0.304565966129303,
-      "reward_std": 0.32997164130210876,
-      "rewards/true_env_reward_fn/mean": 0.304565966129303,
-      "rewards/true_env_reward_fn/std": 0.3299716114997864,
+      "completions/max_length": 133.0,
+      "completions/max_terminated_length": 133.0,
+      "completions/mean_length": 66.125,
+      "completions/mean_terminated_length": 66.125,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2666764855384827,
+      "epoch": 4.170731707317073,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06263021379709244,
+      "kl": 5.531543774850434e-05,
+      "learning_rate": 7.275640035440044e-07,
+      "loss": -0.004293827340006828,
+      "num_tokens": 4215315.0,
+      "reward": 0.5377860069274902,
+      "reward_std": 0.22855143249034882,
+      "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+      "rewards/true_env_reward_fn/std": 0.2285514622926712,
       "step": 171,
-      "step_time": 5.0436168590003945
+      "step_time": 8.973740739000277
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4804,26 +4804,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 71.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1254178285598755,
-      "epoch": 1.3983739837398375,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15058821439743042,
-      "kl": 1.8407325114822015e-05,
-      "learning_rate": 5.192791644186662e-07,
-      "loss": 0.025478817522525787,
-      "num_tokens": 706411.0,
-      "reward": 0.7279239892959595,
-      "reward_std": 0.11376125365495682,
-      "rewards/true_env_reward_fn/mean": 0.7279239892959595,
-      "rewards/true_env_reward_fn/std": 0.11376123130321503,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 69.0,
+      "completions/mean_terminated_length": 69.0,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2872737050056458,
+      "epoch": 4.195121951219512,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09720690548419952,
+      "kl": 8.13291462691268e-05,
+      "learning_rate": 7.237653468464755e-07,
+      "loss": 0.015124019235372543,
+      "num_tokens": 4235707.0,
+      "reward": 0.5968735814094543,
+      "reward_std": 0.2860201299190521,
+      "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+      "rewards/true_env_reward_fn/std": 0.28602010011672974,
       "step": 172,
-      "step_time": 3.2261944119982218
+      "step_time": 10.39117795600032
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4832,26 +4832,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 65.625,
-      "completions/mean_terminated_length": 65.625,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.0894773602485657,
-      "epoch": 1.4065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1687694787979126,
-      "kl": 2.6046765015053097e-05,
-      "learning_rate": 5.156610312374013e-07,
-      "loss": -0.056941211223602295,
-      "num_tokens": 711212.0,
-      "reward": 0.4907146692276001,
-      "reward_std": 0.3376546800136566,
-      "rewards/true_env_reward_fn/mean": 0.4907146692276001,
-      "rewards/true_env_reward_fn/std": 0.337654709815979,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 74.77083587646484,
+      "completions/mean_terminated_length": 74.77083587646484,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2326436638832092,
+      "epoch": 4.219512195121951,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.046250198036432266,
+      "kl": 4.305509810365038e-05,
+      "learning_rate": 7.199504706579616e-07,
+      "loss": -0.010809645056724548,
+      "num_tokens": 4262504.0,
+      "reward": 0.46610039472579956,
+      "reward_std": 0.2549833357334137,
+      "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+      "rewards/true_env_reward_fn/std": 0.2549833059310913,
       "step": 173,
-      "step_time": 3.9498180619993946
+      "step_time": 11.8111169230001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4860,26 +4860,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 60.125,
-      "completions/mean_terminated_length": 60.125,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2238691449165344,
-      "epoch": 1.4146341463414633,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19584441184997559,
-      "kl": 4.8285241064149886e-05,
-      "learning_rate": 5.120325416899629e-07,
-      "loss": 0.0766875222325325,
-      "num_tokens": 715409.0,
-      "reward": 0.4593355059623718,
-      "reward_std": 0.3909546136856079,
-      "rewards/true_env_reward_fn/mean": 0.4593355059623718,
-      "rewards/true_env_reward_fn/std": 0.3909546434879303,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 189.0,
+      "completions/mean_length": 77.16667175292969,
+      "completions/mean_terminated_length": 77.16667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.261723816394806,
+      "epoch": 4.2439024390243905,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07238215953111649,
+      "kl": 7.066424404911231e-05,
+      "learning_rate": 7.161196514973734e-07,
+      "loss": 0.04901377111673355,
+      "num_tokens": 4290472.0,
+      "reward": 0.3766266703605652,
+      "reward_std": 0.27605685591697693,
+      "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+      "rewards/true_env_reward_fn/std": 0.27605685591697693,
       "step": 174,
-      "step_time": 4.100519798999812
+      "step_time": 17.211099596999702
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4888,26 +4888,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 60.25,
-      "completions/mean_terminated_length": 60.25,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.1763202548027039,
-      "epoch": 1.4227642276422765,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1448555737733841,
-      "kl": 1.2618989785551094e-05,
-      "learning_rate": 5.08394020673734e-07,
-      "loss": -0.012558378279209137,
-      "num_tokens": 722327.0,
-      "reward": 0.15966665744781494,
-      "reward_std": 0.3235519230365753,
-      "rewards/true_env_reward_fn/mean": 0.15966665744781494,
-      "rewards/true_env_reward_fn/std": 0.32355189323425293,
+      "completions/max_length": 153.0,
+      "completions/max_terminated_length": 153.0,
+      "completions/mean_length": 74.14583587646484,
+      "completions/mean_terminated_length": 74.14583587646484,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.284770429134369,
+      "epoch": 4.2682926829268295,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.03949572518467903,
+      "kl": 3.151347550556238e-05,
+      "learning_rate": 7.12273167039238e-07,
+      "loss": 0.007944343611598015,
+      "num_tokens": 4316687.0,
+      "reward": 0.46209168434143066,
+      "reward_std": 0.1050746738910675,
+      "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+      "rewards/true_env_reward_fn/std": 0.1050746738910675,
       "step": 175,
-      "step_time": 3.875348296000084
+      "step_time": 12.29185969800028
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4916,26 +4916,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2852763533592224,
-      "epoch": 1.4308943089430894,
+      "completions/max_length": 119.0,
+      "completions/max_terminated_length": 119.0,
+      "completions/mean_length": 66.54167175292969,
+      "completions/mean_terminated_length": 66.54167175292969,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.3430605232715607,
+      "epoch": 4.2926829268292686,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22303813695907593,
-      "kl": 4.580334098136518e-05,
-      "learning_rate": 5.047457939843227e-07,
-      "loss": -0.09214464575052261,
-      "num_tokens": 726828.0,
-      "reward": 0.25830498337745667,
-      "reward_std": 0.37860655784606934,
-      "rewards/true_env_reward_fn/mean": 0.25830498337745667,
-      "rewards/true_env_reward_fn/std": 0.37860655784606934,
+      "grad_norm": 0.09476255625486374,
+      "kl": 9.628380030335393e-05,
+      "learning_rate": 7.084112960935715e-07,
+      "loss": -0.006478719413280487,
+      "num_tokens": 4338257.0,
+      "reward": 0.38874804973602295,
+      "reward_std": 0.2893269658088684,
+      "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+      "rewards/true_env_reward_fn/std": 0.289326936006546,
       "step": 176,
-      "step_time": 4.241473076999682
+      "step_time": 10.987576109999736
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4944,26 +4944,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 58.75,
-      "completions/mean_terminated_length": 58.75,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.3609731197357178,
-      "epoch": 1.4390243902439024,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.001985745271668e-05,
-      "kl": 1.4942165307729738e-05,
-      "learning_rate": 5.010881882863893e-07,
-      "loss": 7.44550789022469e-07,
-      "num_tokens": 729930.0,
-      "reward": 0.6203632950782776,
-      "reward_std": 0.11898252367973328,
-      "rewards/true_env_reward_fn/mean": 0.6203632950782776,
-      "rewards/true_env_reward_fn/std": 0.11898253113031387,
+      "completions/max_length": 134.0,
+      "completions/max_terminated_length": 134.0,
+      "completions/mean_length": 67.85417175292969,
+      "completions/mean_terminated_length": 67.85417175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3582488000392914,
+      "epoch": 4.317073170731708,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0771971270442009,
+      "kl": 8.3626783634827e-05,
+      "learning_rate": 7.0453431858567e-07,
+      "loss": 0.032796651124954224,
+      "num_tokens": 4362418.0,
+      "reward": 0.4179423749446869,
+      "reward_std": 0.33730608224868774,
+      "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+      "rewards/true_env_reward_fn/std": 0.33730608224868774,
       "step": 177,
-      "step_time": 3.3902666960002534
+      "step_time": 10.792315139000493
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4972,26 +4972,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 59.125,
-      "completions/mean_terminated_length": 59.125,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.2678966522216797,
-      "epoch": 1.4471544715447155,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15351757407188416,
-      "kl": 1.6737100395403104e-05,
-      "learning_rate": 4.974215310843967e-07,
-      "loss": 0.041131969541311264,
-      "num_tokens": 732155.0,
-      "reward": 0.7803820371627808,
-      "reward_std": 0.08667682856321335,
-      "rewards/true_env_reward_fn/mean": 0.7803820371627808,
-      "rewards/true_env_reward_fn/std": 0.08667682856321335,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 58.208335876464844,
+      "completions/mean_terminated_length": 58.208335876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.1685997247695923,
+      "epoch": 4.341463414634147,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10942906141281128,
+      "kl": 5.940973733231658e-05,
+      "learning_rate": 7.006425155358195e-07,
+      "loss": 0.12245109677314758,
+      "num_tokens": 4381716.0,
+      "reward": 0.5206946134567261,
+      "reward_std": 0.31685587763786316,
+      "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+      "rewards/true_env_reward_fn/std": 0.3168558180332184,
       "step": 178,
-      "step_time": 3.637111981999624
+      "step_time": 12.399353334999887
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5000,26 +5000,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 55.875,
-      "completions/mean_terminated_length": 55.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.146271526813507,
-      "epoch": 1.4552845528455285,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18466196954250336,
-      "kl": 3.719841197380447e-05,
-      "learning_rate": 4.937461506932859e-07,
-      "loss": 0.029051154851913452,
-      "num_tokens": 735418.0,
-      "reward": 0.40377071499824524,
-      "reward_std": 0.28345924615859985,
-      "rewards/true_env_reward_fn/mean": 0.40377071499824524,
-      "rewards/true_env_reward_fn/std": 0.28345924615859985,
+      "completions/max_length": 180.0,
+      "completions/max_terminated_length": 180.0,
+      "completions/mean_length": 77.6875,
+      "completions/mean_terminated_length": 77.6875,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.302725076675415,
+      "epoch": 4.365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06826934218406677,
+      "kl": 3.6839799577137455e-05,
+      "learning_rate": 6.967361690389258e-07,
+      "loss": -0.03518716245889664,
+      "num_tokens": 4405845.0,
+      "reward": 0.5313202142715454,
+      "reward_std": 0.18632179498672485,
+      "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+      "rewards/true_env_reward_fn/std": 0.18632179498672485,
       "step": 179,
-      "step_time": 3.340555791999577
+      "step_time": 12.45691162600042
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5028,26 +5028,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 170.0,
-      "completions/max_terminated_length": 170.0,
-      "completions/mean_length": 79.75,
-      "completions/mean_terminated_length": 79.75,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.3902945518493652,
-      "epoch": 1.4634146341463414,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1020251139998436,
-      "kl": 1.8220500351162627e-05,
-      "learning_rate": 4.900623762090777e-07,
-      "loss": -0.002344265580177307,
-      "num_tokens": 740540.0,
-      "reward": 0.3590222895145416,
-      "reward_std": 0.12487777322530746,
-      "rewards/true_env_reward_fn/mean": 0.3590222895145416,
-      "rewards/true_env_reward_fn/std": 0.12487778067588806,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 73.39583587646484,
+      "completions/mean_terminated_length": 73.39583587646484,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2715371549129486,
+      "epoch": 4.390243902439025,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05825073644518852,
+      "kl": 7.535525219282135e-05,
+      "learning_rate": 6.928155622440679e-07,
+      "loss": -0.018800390884280205,
+      "num_tokens": 4426576.0,
+      "reward": 0.47252464294433594,
+      "reward_std": 0.33459845185279846,
+      "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+      "rewards/true_env_reward_fn/std": 0.33459845185279846,
       "step": 180,
-      "step_time": 7.219923718001155
+      "step_time": 12.108760526000424
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5056,26 +5056,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 60.875,
-      "completions/mean_terminated_length": 60.875,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.517557680606842,
-      "epoch": 1.4715447154471546,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24192294478416443,
-      "kl": 4.1268089262302965e-05,
-      "learning_rate": 4.863705374794055e-07,
-      "loss": 0.09132950007915497,
-      "num_tokens": 744723.0,
-      "reward": 0.23991000652313232,
-      "reward_std": 0.2837013602256775,
-      "rewards/true_env_reward_fn/mean": 0.23991000652313232,
-      "rewards/true_env_reward_fn/std": 0.2837013602256775,
+      "completions/max_length": 185.0,
+      "completions/max_terminated_length": 185.0,
+      "completions/mean_length": 61.020835876464844,
+      "completions/mean_terminated_length": 61.020835876464844,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.1720183491706848,
+      "epoch": 4.414634146341464,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08154451847076416,
+      "kl": 9.67955465966952e-05,
+      "learning_rate": 6.888809793339728e-07,
+      "loss": 0.016710905358195305,
+      "num_tokens": 4448649.0,
+      "reward": 0.5532544851303101,
+      "reward_std": 0.17937251925468445,
+      "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+      "rewards/true_env_reward_fn/std": 0.17937250435352325,
       "step": 181,
-      "step_time": 3.696339096999509
+      "step_time": 14.16400909000049
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5084,26 +5084,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 66.0,
-      "completions/max_terminated_length": 66.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.249614655971527,
-      "epoch": 1.4796747967479675,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1566745787858963,
-      "kl": 2.6629099920683075e-05,
-      "learning_rate": 4.826709650739811e-07,
-      "loss": 0.003972277045249939,
-      "num_tokens": 748979.0,
-      "reward": 0.4935140311717987,
-      "reward_std": 0.41978561878204346,
-      "rewards/true_env_reward_fn/mean": 0.4935140311717987,
-      "rewards/true_env_reward_fn/std": 0.41978561878204346,
+      "completions/max_length": 193.0,
+      "completions/max_terminated_length": 193.0,
+      "completions/mean_length": 71.9375,
+      "completions/mean_terminated_length": 71.9375,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2416147291660309,
+      "epoch": 4.439024390243903,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05918826535344124,
+      "kl": 6.67227295707562e-05,
+      "learning_rate": 6.849327055044181e-07,
+      "loss": 0.04629965499043465,
+      "num_tokens": 4479382.0,
+      "reward": 0.3571457862854004,
+      "reward_std": 0.3042747676372528,
+      "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+      "rewards/true_env_reward_fn/std": 0.3042747676372528,
       "step": 182,
-      "step_time": 3.316512920000605
+      "step_time": 19.767916835999586
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5112,26 +5112,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.1666916608810425,
-      "epoch": 1.4878048780487805,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001005820304271765,
-      "kl": 1.2246940059412736e-05,
-      "learning_rate": 4.789639902549948e-07,
-      "loss": 6.330609494398232e-07,
-      "num_tokens": 751323.0,
-      "reward": 0.8357743620872498,
-      "reward_std": 0.11130158603191376,
-      "rewards/true_env_reward_fn/mean": 0.8357743620872498,
-      "rewards/true_env_reward_fn/std": 0.11130159348249435,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 70.52083587646484,
+      "completions/mean_terminated_length": 70.52083587646484,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.2693078517913818,
+      "epoch": 4.463414634146342,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06269724667072296,
+      "kl": 4.1979371417255607e-05,
+      "learning_rate": 6.809710269435589e-07,
+      "loss": 0.003845077008008957,
+      "num_tokens": 4501263.0,
+      "reward": 0.5583165884017944,
+      "reward_std": 0.19653278589248657,
+      "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+      "rewards/true_env_reward_fn/std": 0.19653277099132538,
       "step": 183,
-      "step_time": 3.7099916660008603
+      "step_time": 10.428195530999346
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5140,26 +5140,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.3246222138404846,
-      "epoch": 1.4959349593495934,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18140996992588043,
-      "kl": 3.1042441150930244e-05,
-      "learning_rate": 4.752499449474535e-07,
-      "loss": -0.022353097796440125,
-      "num_tokens": 755494.0,
-      "reward": 0.4561777412891388,
-      "reward_std": 0.2439236342906952,
-      "rewards/true_env_reward_fn/mean": 0.4561777412891388,
-      "rewards/true_env_reward_fn/std": 0.24392364919185638,
+      "completions/max_length": 183.0,
+      "completions/max_terminated_length": 183.0,
+      "completions/mean_length": 77.66667175292969,
+      "completions/mean_terminated_length": 77.66667175292969,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.2504103481769562,
+      "epoch": 4.487804878048781,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07646579295396805,
+      "kl": 9.529235103400424e-05,
+      "learning_rate": 6.769962308111839e-07,
+      "loss": 0.007325906306505203,
+      "num_tokens": 4525959.0,
+      "reward": 0.4377995431423187,
+      "reward_std": 0.330658495426178,
+      "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+      "rewards/true_env_reward_fn/std": 0.330658495426178,
       "step": 184,
-      "step_time": 3.7916486710000754
+      "step_time": 16.47171987999991
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5168,26 +5168,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 75.75,
-      "completions/mean_terminated_length": 75.75,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.2625707983970642,
-      "epoch": 1.5040650406504064,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11702944338321686,
-      "kl": 1.503958355897339e-05,
-      "learning_rate": 4.715291617094607e-07,
-      "loss": 0.023916304111480713,
-      "num_tokens": 758432.0,
-      "reward": 0.5015827417373657,
-      "reward_std": 0.1783808320760727,
-      "rewards/true_env_reward_fn/mean": 0.5015827417373657,
-      "rewards/true_env_reward_fn/std": 0.1783808171749115,
+      "completions/max_length": 203.0,
+      "completions/max_terminated_length": 203.0,
+      "completions/mean_length": 71.97917175292969,
+      "completions/mean_terminated_length": 71.97917175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2982739210128784,
+      "epoch": 4.512195121951219,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07417281717061996,
+      "kl": 4.4408230678527616e-05,
+      "learning_rate": 6.730086052179002e-07,
+      "loss": 0.0469856858253479,
+      "num_tokens": 4546310.0,
+      "reward": 0.5151915550231934,
+      "reward_std": 0.22422264516353607,
+      "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+      "rewards/true_env_reward_fn/std": 0.22422264516353607,
       "step": 185,
-      "step_time": 4.226409274000616
+      "step_time": 13.684267182999974
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5196,26 +5196,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 54.0,
-      "completions/mean_terminated_length": 54.0,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 1.1460023522377014,
-      "epoch": 1.5121951219512195,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11851620674133301,
-      "kl": 1.3728345948038623e-05,
-      "learning_rate": 4.678019737024387e-07,
-      "loss": 0.0831337422132492,
-      "num_tokens": 764336.0,
-      "reward": 0.19094166159629822,
-      "reward_std": 0.30934420228004456,
-      "rewards/true_env_reward_fn/mean": 0.19094166159629822,
-      "rewards/true_env_reward_fn/std": 0.30934423208236694,
+      "completions/max_length": 125.0,
+      "completions/max_terminated_length": 125.0,
+      "completions/mean_length": 65.70833587646484,
+      "completions/mean_terminated_length": 65.70833587646484,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.2992590963840485,
+      "epoch": 4.536585365853659,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07500731945037842,
+      "kl": 6.5705213273759e-05,
+      "learning_rate": 6.690084392042513e-07,
+      "loss": -0.006503798067569733,
+      "num_tokens": 4566816.0,
+      "reward": 0.5268750190734863,
+      "reward_std": 0.23048490285873413,
+      "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+      "rewards/true_env_reward_fn/std": 0.23048490285873413,
       "step": 186,
-      "step_time": 3.563357556000483
+      "step_time": 10.281018189999031
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5224,26 +5224,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 58.5,
-      "completions/mean_terminated_length": 58.5,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.1291148662567139,
-      "epoch": 1.5203252032520327,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.864256960805506e-05,
-      "kl": 1.081683785741916e-05,
-      "learning_rate": 4.6406871466129704e-07,
-      "loss": 5.412177870312007e-07,
-      "num_tokens": 766608.0,
-      "reward": 0.8357743620872498,
-      "reward_std": 0.11130158603191376,
-      "rewards/true_env_reward_fn/mean": 0.8357743620872498,
-      "rewards/true_env_reward_fn/std": 0.11130159348249435,
+      "completions/max_length": 355.0,
+      "completions/max_terminated_length": 355.0,
+      "completions/mean_length": 66.91667175292969,
+      "completions/mean_terminated_length": 66.91667175292969,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.3209801018238068,
+      "epoch": 4.560975609756097,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.082595095038414,
+      "kl": 8.261651692009764e-05,
+      "learning_rate": 6.649960227197647e-07,
+      "loss": 0.0495578795671463,
+      "num_tokens": 4589204.0,
+      "reward": 0.4924369752407074,
+      "reward_std": 0.37204882502555847,
+      "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+      "rewards/true_env_reward_fn/std": 0.37204885482788086,
       "step": 187,
-      "step_time": 3.534869859002356
+      "step_time": 24.351223329000277
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5252,26 +5252,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 189.0,
-      "completions/max_terminated_length": 189.0,
-      "completions/mean_length": 92.0,
-      "completions/mean_terminated_length": 92.0,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.3645328283309937,
-      "epoch": 1.5284552845528454,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18168245255947113,
-      "kl": 2.6857565899263136e-05,
-      "learning_rate": 4.6032971886454956e-07,
-      "loss": -0.005156125873327255,
-      "num_tokens": 774496.0,
-      "reward": 0.11249999701976776,
-      "reward_std": 0.20856082439422607,
-      "rewards/true_env_reward_fn/mean": 0.11249999701976776,
-      "rewards/true_env_reward_fn/std": 0.20856082439422607,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 62.9375,
+      "completions/mean_terminated_length": 62.9375,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.3033248782157898,
+      "epoch": 4.585365853658536,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0885128378868103,
+      "kl": 0.000125904198284843,
+      "learning_rate": 6.609716466019355e-07,
+      "loss": 0.04691624641418457,
+      "num_tokens": 4622361.0,
+      "reward": 0.16021786630153656,
+      "reward_std": 0.24737857282161713,
+      "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+      "rewards/true_env_reward_fn/std": 0.24737857282161713,
       "step": 188,
-      "step_time": 8.34005261099992
+      "step_time": 10.644911742000204
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5280,26 +5280,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 58.75,
-      "completions/mean_terminated_length": 58.75,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.1495982110500336,
-      "epoch": 1.5365853658536586,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14486820995807648,
-      "kl": 1.3202762147557223e-05,
-      "learning_rate": 4.5658532110438337e-07,
-      "loss": -0.0010610297322273254,
-      "num_tokens": 777186.0,
-      "reward": 0.5879127383232117,
-      "reward_std": 0.05142820253968239,
-      "rewards/true_env_reward_fn/mean": 0.5879127383232117,
-      "rewards/true_env_reward_fn/std": 0.05142820253968239,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 56.25,
+      "completions/mean_terminated_length": 56.25,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2344954907894135,
+      "epoch": 4.609756097560975,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07845015823841095,
+      "kl": 8.022368183446815e-05,
+      "learning_rate": 6.569356025551454e-07,
+      "loss": 0.03257204219698906,
+      "num_tokens": 4649381.0,
+      "reward": 0.3547590970993042,
+      "reward_std": 0.23717434704303741,
+      "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+      "rewards/true_env_reward_fn/std": 0.23717434704303741,
       "step": 189,
-      "step_time": 3.4856022139993
+      "step_time": 9.579594637999435
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5308,26 +5308,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 65.375,
-      "completions/mean_terminated_length": 65.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.241140365600586,
-      "epoch": 1.5447154471544715,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001250067143701017,
-      "kl": 1.5482702110602986e-05,
-      "learning_rate": 4.52835856656681e-07,
-      "loss": 7.80837922320643e-07,
-      "num_tokens": 779965.0,
-      "reward": 0.6861198544502258,
-      "reward_std": 0.08807206153869629,
-      "rewards/true_env_reward_fn/mean": 0.6861198544502258,
-      "rewards/true_env_reward_fn/std": 0.08807206153869629,
+      "completions/max_length": 146.0,
+      "completions/max_terminated_length": 146.0,
+      "completions/mean_length": 62.0,
+      "completions/mean_terminated_length": 62.0,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.258386880159378,
+      "epoch": 4.634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07976282387971878,
+      "kl": 0.00011880166857736185,
+      "learning_rate": 6.528881831295188e-07,
+      "loss": -0.02127165161073208,
+      "num_tokens": 4674357.0,
+      "reward": 0.4042941927909851,
+      "reward_std": 0.2737519145011902,
+      "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+      "rewards/true_env_reward_fn/std": 0.2737519443035126,
       "step": 190,
-      "step_time": 3.904181735999373
+      "step_time": 11.495368679999501
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5336,26 +5336,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.2944807410240173,
-      "epoch": 1.5528455284552845,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2030111849308014,
-      "kl": 3.585523518268019e-05,
-      "learning_rate": 4.490816612509991e-07,
-      "loss": 0.0143373291939497,
-      "num_tokens": 786140.0,
-      "reward": 0.35173332691192627,
-      "reward_std": 0.18115806579589844,
-      "rewards/true_env_reward_fn/mean": 0.35173332691192627,
-      "rewards/true_env_reward_fn/std": 0.18115808069705963,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 64.25,
+      "completions/mean_terminated_length": 64.25,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2503610253334045,
+      "epoch": 4.658536585365853,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08848423510789871,
+      "kl": 0.00012839957071264507,
+      "learning_rate": 6.488296816997173e-07,
+      "loss": 0.04479096084833145,
+      "num_tokens": 4694425.0,
+      "reward": 0.5075992345809937,
+      "reward_std": 0.2306082397699356,
+      "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+      "rewards/true_env_reward_fn/std": 0.2306082397699356,
       "step": 191,
-      "step_time": 4.005758510000305
+      "step_time": 7.909104242000012
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5364,26 +5364,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 58.0,
-      "completions/mean_terminated_length": 58.0,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2177271246910095,
-      "epoch": 1.5609756097560976,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11188288033008575,
-      "kl": 2.3622495064046234e-05,
-      "learning_rate": 4.45323071040508e-07,
-      "loss": -0.02369789034128189,
-      "num_tokens": 790424.0,
-      "reward": 0.4888629913330078,
-      "reward_std": 0.23310808837413788,
-      "rewards/true_env_reward_fn/mean": 0.4888629913330078,
-      "rewards/true_env_reward_fn/std": 0.23310808837413788,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 60.520835876464844,
+      "completions/mean_terminated_length": 60.520835876464844,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.266694962978363,
+      "epoch": 4.682926829268292,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0782349705696106,
+      "kl": 8.841241015034029e-05,
+      "learning_rate": 6.447603924436743e-07,
+      "loss": 0.030293334275484085,
+      "num_tokens": 4720074.0,
+      "reward": 0.42062053084373474,
+      "reward_std": 0.17757493257522583,
+      "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+      "rewards/true_env_reward_fn/std": 0.17757493257522583,
       "step": 192,
-      "step_time": 3.2518814809991454
+      "step_time": 10.043768619000275
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5392,26 +5392,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 65.625,
-      "completions/mean_terminated_length": 65.625,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.192966103553772,
-      "epoch": 1.5691056910569106,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1378823071718216,
-      "kl": 1.7358055174554465e-05,
-      "learning_rate": 4.4156042257189143e-07,
-      "loss": 0.06256310641765594,
-      "num_tokens": 794521.0,
-      "reward": 0.5219699740409851,
-      "reward_std": 0.06214587390422821,
-      "rewards/true_env_reward_fn/mean": 0.5219699740409851,
-      "rewards/true_env_reward_fn/std": 0.062145888805389404,
+      "completions/max_length": 106.0,
+      "completions/max_terminated_length": 106.0,
+      "completions/mean_length": 63.60416793823242,
+      "completions/mean_terminated_length": 63.60416793823242,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.23249152302742,
+      "epoch": 4.7073170731707314,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09907371550798416,
+      "kl": 0.0001827988016884774,
+      "learning_rate": 6.406806103212724e-07,
+      "loss": 0.010011034086346626,
+      "num_tokens": 4746719.0,
+      "reward": 0.426013708114624,
+      "reward_std": 0.3213046193122864,
+      "rewards/true_env_reward_fn/mean": 0.426013708114624,
+      "rewards/true_env_reward_fn/std": 0.3213046193122864,
       "step": 193,
-      "step_time": 4.222739491999164
+      "step_time": 10.19648474899941
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5420,26 +5420,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 63.125,
-      "completions/mean_terminated_length": 63.125,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.1770159006118774,
-      "epoch": 1.5772357723577235,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19550059735774994,
-      "kl": 2.4871268578863237e-05,
-      "learning_rate": 4.377940527552125e-07,
-      "loss": 0.05841376632452011,
-      "num_tokens": 798194.0,
-      "reward": 0.42302167415618896,
-      "reward_std": 0.2911272346973419,
-      "rewards/true_env_reward_fn/mean": 0.42302167415618896,
-      "rewards/true_env_reward_fn/std": 0.2911272644996643,
+      "completions/max_length": 152.0,
+      "completions/max_terminated_length": 152.0,
+      "completions/mean_length": 64.0625,
+      "completions/mean_terminated_length": 64.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2059581279754639,
+      "epoch": 4.7317073170731705,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08840472996234894,
+      "kl": 0.00012012650222459342,
+      "learning_rate": 6.365906310529629e-07,
+      "loss": -0.034412819892168045,
+      "num_tokens": 4780578.0,
+      "reward": 0.28782567381858826,
+      "reward_std": 0.3653683662414551,
+      "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+      "rewards/true_env_reward_fn/std": 0.3653683662414551,
       "step": 194,
-      "step_time": 3.8170270639984665
+      "step_time": 14.372816425000565
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5448,26 +5448,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 66.5,
-      "completions/mean_terminated_length": 66.5,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.5244255661964417,
-      "epoch": 1.5853658536585367,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1872202306985855,
-      "kl": 2.5990483663917985e-05,
-      "learning_rate": 4.340242988337462e-07,
-      "loss": -0.044112429022789,
-      "num_tokens": 802802.0,
-      "reward": 0.33745431900024414,
-      "reward_std": 0.22955451905727386,
-      "rewards/true_env_reward_fn/mean": 0.33745431900024414,
-      "rewards/true_env_reward_fn/std": 0.22955450415611267,
+      "completions/max_length": 140.0,
+      "completions/max_terminated_length": 140.0,
+      "completions/mean_length": 77.10417175292969,
+      "completions/mean_terminated_length": 77.10417175292969,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3157014548778534,
+      "epoch": 4.7560975609756095,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07452306151390076,
+      "kl": 8.27656508590735e-05,
+      "learning_rate": 6.32490751098331e-07,
+      "loss": -0.004008886404335499,
+      "num_tokens": 4809119.0,
+      "reward": 0.4407285451889038,
+      "reward_std": 0.23340703547000885,
+      "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+      "rewards/true_env_reward_fn/std": 0.23340705037117004,
       "step": 195,
-      "step_time": 4.27381555500142
+      "step_time": 11.282298853000157
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5476,26 +5476,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 69.875,
-      "completions/mean_terminated_length": 69.875,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.345891296863556,
-      "epoch": 1.5934959349593496,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16891054809093475,
-      "kl": 2.7261638024356216e-05,
-      "learning_rate": 4.3025149835378275e-07,
-      "loss": -0.139386385679245,
-      "num_tokens": 807881.0,
-      "reward": 0.3240283131599426,
-      "reward_std": 0.2803676128387451,
-      "rewards/true_env_reward_fn/mean": 0.3240283131599426,
-      "rewards/true_env_reward_fn/std": 0.2803676128387451,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 61.458335876464844,
+      "completions/mean_terminated_length": 61.458335876464844,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2990687191486359,
+      "epoch": 4.780487804878049,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07655440270900726,
+      "kl": 0.0001329305760009447,
+      "learning_rate": 6.283812676346063e-07,
+      "loss": 0.024208612740039825,
+      "num_tokens": 4835557.0,
+      "reward": 0.42621374130249023,
+      "reward_std": 0.28145232796669006,
+      "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+      "rewards/true_env_reward_fn/std": 0.28145232796669006,
       "step": 196,
-      "step_time": 4.981287381999209
+      "step_time": 8.65745804199969
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5504,26 +5504,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 61.0,
-      "completions/max_terminated_length": 61.0,
-      "completions/mean_length": 56.375,
-      "completions/mean_terminated_length": 56.375,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.246802031993866,
-      "epoch": 1.6016260162601625,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2096729576587677,
-      "kl": 3.5958016269432846e-05,
-      "learning_rate": 4.2647598913440264e-07,
-      "loss": -0.02941281348466873,
-      "num_tokens": 812500.0,
-      "reward": 0.4126526415348053,
-      "reward_std": 0.36393746733665466,
-      "rewards/true_env_reward_fn/mean": 0.4126526415348053,
-      "rewards/true_env_reward_fn/std": 0.36393749713897705,
+      "completions/max_length": 90.0,
+      "completions/max_terminated_length": 90.0,
+      "completions/mean_length": 56.333335876464844,
+      "completions/mean_terminated_length": 56.333335876464844,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.148613840341568,
+      "epoch": 4.804878048780488,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08951497822999954,
+      "kl": 5.616615226244903e-05,
+      "learning_rate": 6.242624785351235e-07,
+      "loss": 0.04646766185760498,
+      "num_tokens": 4861629.0,
+      "reward": 0.3379192352294922,
+      "reward_std": 0.3291850686073303,
+      "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+      "rewards/true_env_reward_fn/std": 0.3291850984096527,
       "step": 197,
-      "step_time": 3.101726017999681
+      "step_time": 9.615110594999805
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5532,26 +5532,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 66.75,
-      "completions/mean_terminated_length": 66.75,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.4369062185287476,
-      "epoch": 1.6097560975609757,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1862105429172516,
-      "kl": 4.7646244638599455e-05,
-      "learning_rate": 4.2269810923722965e-07,
-      "loss": 0.0521145761013031,
-      "num_tokens": 818718.0,
-      "reward": 0.26869943737983704,
-      "reward_std": 0.1483483463525772,
-      "rewards/true_env_reward_fn/mean": 0.26869943737983704,
-      "rewards/true_env_reward_fn/std": 0.1483483612537384,
+      "completions/max_length": 139.0,
+      "completions/max_terminated_length": 139.0,
+      "completions/mean_length": 82.20833587646484,
+      "completions/mean_terminated_length": 82.20833587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.0347496271133423,
+      "epoch": 4.829268292682927,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.038518842309713364,
+      "kl": 1.8024265045823995e-05,
+      "learning_rate": 6.201346823477302e-07,
+      "loss": 0.005069371312856674,
+      "num_tokens": 4880671.0,
+      "reward": 0.5492597818374634,
+      "reward_std": 0.20638948678970337,
+      "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+      "rewards/true_env_reward_fn/std": 0.20638947188854218,
       "step": 198,
-      "step_time": 4.287780451000799
+      "step_time": 10.832162847999825
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5560,26 +5560,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 59.25,
-      "completions/mean_terminated_length": 59.25,
-      "completions/min_length": 21.0,
-      "completions/min_terminated_length": 21.0,
-      "entropy": 1.192937195301056,
-      "epoch": 1.6178861788617886,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2013663798570633,
-      "kl": 1.8760739294521045e-05,
-      "learning_rate": 4.189181969361588e-07,
-      "loss": 0.07236722111701965,
-      "num_tokens": 825728.0,
-      "reward": 0.23110000789165497,
-      "reward_std": 0.23212090134620667,
-      "rewards/true_env_reward_fn/mean": 0.23110000789165497,
-      "rewards/true_env_reward_fn/std": 0.23212090134620667,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 65.60417175292969,
+      "completions/mean_terminated_length": 65.60417175292969,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2905827760696411,
+      "epoch": 4.853658536585366,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0785721018910408,
+      "kl": 7.324252146645449e-05,
+      "learning_rate": 6.159981782731473e-07,
+      "loss": -0.021710166707634926,
+      "num_tokens": 4904516.0,
+      "reward": 0.4835298955440521,
+      "reward_std": 0.27475905418395996,
+      "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+      "rewards/true_env_reward_fn/std": 0.27475905418395996,
       "step": 199,
-      "step_time": 4.708717262998107
+      "step_time": 10.94759418800004
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5588,26 +5588,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 72.25,
-      "completions/mean_terminated_length": 72.25,
-      "completions/min_length": 62.0,
-      "completions/min_terminated_length": 62.0,
-      "entropy": 1.3792839050292969,
-      "epoch": 1.6260162601626016,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.7985117463395e-05,
-      "kl": 1.4280476534622721e-05,
-      "learning_rate": 4.1513659068706814e-07,
-      "loss": 7.153485626076872e-07,
-      "num_tokens": 830318.0,
-      "reward": 0.5384680032730103,
-      "reward_std": 0.06977442651987076,
-      "rewards/true_env_reward_fn/mean": 0.5384680032730103,
-      "rewards/true_env_reward_fn/std": 0.06977442651987076,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 70.20833587646484,
+      "completions/mean_terminated_length": 70.20833587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.284969449043274,
+      "epoch": 4.878048780487805,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0855984017252922,
+      "kl": 4.8397108912467957e-05,
+      "learning_rate": 6.118532661432811e-07,
+      "loss": 0.03759150952100754,
+      "num_tokens": 4928134.0,
+      "reward": 0.5413213968276978,
+      "reward_std": 0.18470171093940735,
+      "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+      "rewards/true_env_reward_fn/std": 0.18470169603824615,
       "step": 200,
-      "step_time": 3.9055351140013954
+      "step_time": 10.322844021000492
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5616,54 +5616,54 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 64.375,
-      "completions/mean_terminated_length": 64.375,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.0505937337875366,
-      "epoch": 1.6341463414634148,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13188514113426208,
-      "kl": 2.842090088961413e-05,
-      "learning_rate": 4.1135362909751326e-07,
-      "loss": -0.0017508119344711304,
-      "num_tokens": 834933.0,
-      "reward": 0.36545002460479736,
-      "reward_std": 0.24526984989643097,
-      "rewards/true_env_reward_fn/mean": 0.36545002460479736,
-      "rewards/true_env_reward_fn/std": 0.24526986479759216,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 62.083335876464844,
+      "completions/mean_terminated_length": 62.083335876464844,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.319727510213852,
+      "epoch": 4.902439024390244,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08774503320455551,
+      "kl": 6.86226876496221e-05,
+      "learning_rate": 6.077002463994907e-07,
+      "loss": 0.058894164860248566,
+      "num_tokens": 4956466.0,
+      "reward": 0.41384777426719666,
+      "reward_std": 0.17193447053432465,
+      "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+      "rewards/true_env_reward_fn/std": 0.17193445563316345,
       "step": 201,
-      "step_time": 3.895525625997834
+      "step_time": 10.146928047999609
     },
     {
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 163.0,
-      "completions/max_terminated_length": 163.0,
-      "completions/mean_length": 76.375,
-      "completions/mean_terminated_length": 76.375,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.3750707507133484,
-      "epoch": 1.6422764227642277,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1798955649137497,
-      "kl": 2.5428611479583196e-05,
-      "learning_rate": 4.075696508964076e-07,
-      "loss": 0.1843666434288025,
-      "num_tokens": 838076.0,
-      "reward": 0.44641831517219543,
-      "reward_std": 0.30040720105171204,
-      "rewards/true_env_reward_fn/mean": 0.44641831517219543,
-      "rewards/true_env_reward_fn/std": 0.30040720105171204,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 100.0,
+      "completions/max_terminated_length": 100.0,
+      "completions/mean_length": 59.395835876464844,
+      "completions/mean_terminated_length": 59.395835876464844,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.1935442388057709,
+      "epoch": 4.926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08328510820865631,
+      "kl": 7.948942857183283e-05,
+      "learning_rate": 6.035394200708103e-07,
+      "loss": 0.045964501798152924,
+      "num_tokens": 4978733.0,
+      "reward": 0.4896667003631592,
+      "reward_std": 0.3830767571926117,
+      "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+      "rewards/true_env_reward_fn/std": 0.3830767571926117,
       "step": 202,
-      "step_time": 6.74463491500137
+      "step_time": 10.025533761000133
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5672,26 +5672,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 132.0,
-      "completions/max_terminated_length": 132.0,
-      "completions/mean_length": 79.125,
-      "completions/mean_terminated_length": 79.125,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.3369249105453491,
-      "epoch": 1.6504065040650406,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11249764263629913,
-      "kl": 1.1453126717242412e-05,
-      "learning_rate": 4.0378499490369267e-07,
-      "loss": -0.08210685849189758,
-      "num_tokens": 841933.0,
-      "reward": 0.3737962245941162,
-      "reward_std": 0.13184049725532532,
-      "rewards/true_env_reward_fn/mean": 0.3737962245941162,
-      "rewards/true_env_reward_fn/std": 0.13184049725532532,
+      "completions/max_length": 98.0,
+      "completions/max_terminated_length": 98.0,
+      "completions/mean_length": 62.22916793823242,
+      "completions/mean_terminated_length": 62.22916793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.2698631286621094,
+      "epoch": 4.951219512195122,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07158155739307404,
+      "kl": 8.286665615742095e-05,
+      "learning_rate": 5.993710887521302e-07,
+      "loss": -0.03551984950900078,
+      "num_tokens": 5009440.0,
+      "reward": 0.45639634132385254,
+      "reward_std": 0.1837630569934845,
+      "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+      "rewards/true_env_reward_fn/std": 0.1837630420923233,
       "step": 203,
-      "step_time": 5.5639925510004105
+      "step_time": 9.823523802999716
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5700,26 +5700,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 64.25,
-      "completions/mean_terminated_length": 64.25,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 76.25,
+      "completions/mean_terminated_length": 76.25,
       "completions/min_length": 37.0,
       "completions/min_terminated_length": 37.0,
-      "entropy": 1.5737199783325195,
-      "epoch": 1.6585365853658538,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19394491612911224,
-      "kl": 5.05317857459886e-05,
-      "learning_rate": 4e-07,
-      "loss": 0.05622926354408264,
-      "num_tokens": 847711.0,
-      "reward": 0.10725000500679016,
-      "reward_std": 0.19980257749557495,
-      "rewards/true_env_reward_fn/mean": 0.10725000500679016,
-      "rewards/true_env_reward_fn/std": 0.19980257749557495,
+      "entropy": 1.2825455367565155,
+      "epoch": 4.975609756097561,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07431793212890625,
+      "kl": 7.247529447340639e-05,
+      "learning_rate": 5.951955545823342e-07,
+      "loss": -0.02418600022792816,
+      "num_tokens": 5036036.0,
+      "reward": 0.4476773142814636,
+      "reward_std": 0.20447416603565216,
+      "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+      "rewards/true_env_reward_fn/std": 0.20447418093681335,
       "step": 204,
-      "step_time": 4.918089437000162
+      "step_time": 11.960790695000014
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5728,26 +5728,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 60.5,
-      "completions/mean_terminated_length": 60.5,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.119917094707489,
-      "epoch": 1.6666666666666665,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10553985089063644,
-      "kl": 1.2793303994840244e-05,
-      "learning_rate": 3.9621500509630725e-07,
-      "loss": -0.011355768889188766,
-      "num_tokens": 849519.0,
-      "reward": 0.6593211889266968,
-      "reward_std": 0.11862105131149292,
-      "rewards/true_env_reward_fn/mean": 0.6593211889266968,
-      "rewards/true_env_reward_fn/std": 0.11862105131149292,
+      "completions/max_length": 119.0,
+      "completions/max_terminated_length": 119.0,
+      "completions/mean_length": 69.0,
+      "completions/mean_terminated_length": 69.0,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.2438389956951141,
+      "epoch": 5.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06833480298519135,
+      "kl": 5.646793260893901e-05,
+      "learning_rate": 5.91013120222401e-07,
+      "loss": 0.0021926667541265488,
+      "num_tokens": 5063228.0,
+      "reward": 0.4300723075866699,
+      "reward_std": 0.12116922438144684,
+      "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+      "rewards/true_env_reward_fn/std": 0.12116922438144684,
       "step": 205,
-      "step_time": 4.127652793999005
+      "step_time": 9.874485716000436
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5756,26 +5756,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 108.0,
-      "completions/max_terminated_length": 108.0,
-      "completions/mean_length": 75.125,
-      "completions/mean_terminated_length": 75.125,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.3856809735298157,
-      "epoch": 1.6747967479674797,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1911258101463318,
-      "kl": 4.2569914512569085e-05,
-      "learning_rate": 3.9243034910359247e-07,
-      "loss": 0.10561336576938629,
-      "num_tokens": 854156.0,
-      "reward": 0.2516202926635742,
-      "reward_std": 0.22357939183712006,
-      "rewards/true_env_reward_fn/mean": 0.2516202926635742,
-      "rewards/true_env_reward_fn/std": 0.22357939183712006,
+      "completions/max_length": 205.0,
+      "completions/max_terminated_length": 205.0,
+      "completions/mean_length": 66.1875,
+      "completions/mean_terminated_length": 66.1875,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2645181119441986,
+      "epoch": 5.024390243902439,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08248262107372284,
+      "kl": 9.333990783488844e-05,
+      "learning_rate": 5.868240888334652e-07,
+      "loss": 0.12253005057573318,
+      "num_tokens": 5093373.0,
+      "reward": 0.28542038798332214,
+      "reward_std": 0.326623797416687,
+      "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+      "rewards/true_env_reward_fn/std": 0.326623797416687,
       "step": 206,
-      "step_time": 4.714620994000143
+      "step_time": 15.706792760000553
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5784,26 +5784,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.4012945890426636,
-      "epoch": 1.6829268292682928,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14226125180721283,
-      "kl": 5.4918069963605376e-05,
-      "learning_rate": 3.886463709024868e-07,
-      "loss": 0.012949362397193909,
-      "num_tokens": 856151.0,
-      "reward": 0.6204804182052612,
-      "reward_std": 0.44673967361450195,
-      "rewards/true_env_reward_fn/mean": 0.6204804182052612,
-      "rewards/true_env_reward_fn/std": 0.44673967361450195,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 67.16667175292969,
+      "completions/mean_terminated_length": 67.16667175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.34083291888237,
+      "epoch": 5.048780487804878,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07931552827358246,
+      "kl": 0.00010715152166085318,
+      "learning_rate": 5.826287640548424e-07,
+      "loss": 0.0005592256784439087,
+      "num_tokens": 5118933.0,
+      "reward": 0.5035215616226196,
+      "reward_std": 0.304157018661499,
+      "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+      "rewards/true_env_reward_fn/std": 0.30415698885917664,
       "step": 207,
-      "step_time": 4.094810713999323
+      "step_time": 10.04168460900064
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5812,26 +5812,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 67.625,
-      "completions/mean_terminated_length": 67.625,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.3708943128585815,
-      "epoch": 1.6910569105691056,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12903675436973572,
-      "kl": 1.4145812656352064e-05,
-      "learning_rate": 3.8486340931293187e-07,
-      "loss": -0.07838249206542969,
-      "num_tokens": 859380.0,
-      "reward": 0.6400156021118164,
-      "reward_std": 0.1022576317191124,
-      "rewards/true_env_reward_fn/mean": 0.6400156021118164,
-      "rewards/true_env_reward_fn/std": 0.1022576317191124,
+      "completions/max_length": 195.0,
+      "completions/max_terminated_length": 195.0,
+      "completions/mean_length": 73.4375,
+      "completions/mean_terminated_length": 73.4375,
+      "completions/min_length": 11.0,
+      "completions/min_terminated_length": 11.0,
+      "entropy": 1.1821868121623993,
+      "epoch": 5.073170731707317,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07619761675596237,
+      "kl": 0.00013189401397539768,
+      "learning_rate": 5.784274499820213e-07,
+      "loss": -0.011967688798904419,
+      "num_tokens": 5148682.0,
+      "reward": 0.36097532510757446,
+      "reward_std": 0.29208436608314514,
+      "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+      "rewards/true_env_reward_fn/std": 0.29208436608314514,
       "step": 208,
-      "step_time": 4.297900428997309
+      "step_time": 15.226898961000188
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5840,26 +5840,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 56.75,
-      "completions/mean_terminated_length": 56.75,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2782961130142212,
-      "epoch": 1.6991869918699187,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12437833100557327,
-      "kl": 1.4473939245362999e-05,
-      "learning_rate": 3.8108180306384135e-07,
-      "loss": -0.036324724555015564,
-      "num_tokens": 862714.0,
-      "reward": 0.43419933319091797,
-      "reward_std": 0.15345513820648193,
-      "rewards/true_env_reward_fn/mean": 0.43419933319091797,
-      "rewards/true_env_reward_fn/std": 0.15345513820648193,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 256.0,
+      "completions/mean_length": 75.39583587646484,
+      "completions/mean_terminated_length": 75.39583587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2636725008487701,
+      "epoch": 5.097560975609756,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06586338579654694,
+      "kl": 1.9426477138040354e-05,
+      "learning_rate": 5.742204511446203e-07,
+      "loss": -0.047095417976379395,
+      "num_tokens": 5173205.0,
+      "reward": 0.5261925458908081,
+      "reward_std": 0.2689943313598633,
+      "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+      "rewards/true_env_reward_fn/std": 0.2689943313598633,
       "step": 209,
-      "step_time": 3.47861851900052
+      "step_time": 23.690397457000017
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5867,27 +5867,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 512.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 123.875,
-      "completions/mean_terminated_length": 68.42857360839844,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.5170292258262634,
-      "epoch": 1.7073170731707317,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 96.0,
+      "completions/max_terminated_length": 96.0,
+      "completions/mean_length": 54.3125,
+      "completions/mean_terminated_length": 54.3125,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.2415392696857452,
+      "epoch": 5.121951219512195,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1543467938899994,
-      "kl": 2.3121557205740828e-05,
-      "learning_rate": 3.7730189076277037e-07,
-      "loss": -0.3810324966907501,
-      "num_tokens": 869789.0,
-      "reward": 0.3795333504676819,
-      "reward_std": 0.13815106451511383,
-      "rewards/true_env_reward_fn/mean": 0.3795333504676819,
-      "rewards/true_env_reward_fn/std": 0.13815106451511383,
+      "grad_norm": 0.0990133062005043,
+      "kl": 7.119746442185715e-05,
+      "learning_rate": 5.700080724843146e-07,
+      "loss": -0.030588299036026,
+      "num_tokens": 5199892.0,
+      "reward": 0.3418487012386322,
+      "reward_std": 0.36353805661201477,
+      "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+      "rewards/true_env_reward_fn/std": 0.3635380268096924,
       "step": 210,
-      "step_time": 20.698896928999602
+      "step_time": 9.864614251999683
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5896,26 +5896,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.0,
-      "completions/max_terminated_length": 123.0,
-      "completions/mean_length": 69.875,
-      "completions/mean_terminated_length": 69.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.2324861884117126,
-      "epoch": 1.7154471544715446,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11762262135744095,
-      "kl": 1.9743249595194357e-05,
-      "learning_rate": 3.735240108655973e-07,
-      "loss": -0.09104303270578384,
-      "num_tokens": 872604.0,
-      "reward": 0.5916227102279663,
-      "reward_std": 0.1752101182937622,
-      "rewards/true_env_reward_fn/mean": 0.5916227102279663,
-      "rewards/true_env_reward_fn/std": 0.1752101480960846,
+      "completions/max_length": 135.0,
+      "completions/max_terminated_length": 135.0,
+      "completions/mean_length": 62.520835876464844,
+      "completions/mean_terminated_length": 62.520835876464844,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2508135735988617,
+      "epoch": 5.146341463414634,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.10835102945566177,
+      "kl": 0.00012395972225931473,
+      "learning_rate": 5.657906193327324e-07,
+      "loss": 0.08621911704540253,
+      "num_tokens": 5220141.0,
+      "reward": 0.4987506866455078,
+      "reward_std": 0.27843451499938965,
+      "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+      "rewards/true_env_reward_fn/std": 0.27843451499938965,
       "step": 211,
-      "step_time": 5.218213289999767
+      "step_time": 9.983622502000344
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5924,26 +5924,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.4448966979980469,
-      "epoch": 1.7235772357723578,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1924143135547638,
-      "kl": 3.186432604707079e-05,
-      "learning_rate": 3.697485016462174e-07,
-      "loss": 0.025449808686971664,
-      "num_tokens": 877921.0,
-      "reward": 0.1028124988079071,
-      "reward_std": 0.2254277467727661,
-      "rewards/true_env_reward_fn/mean": 0.1028124988079071,
-      "rewards/true_env_reward_fn/std": 0.22542773187160492,
+      "completions/max_length": 265.0,
+      "completions/max_terminated_length": 265.0,
+      "completions/mean_length": 66.02083587646484,
+      "completions/mean_terminated_length": 66.02083587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.1611086130142212,
+      "epoch": 5.170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08423946797847748,
+      "kl": 4.4347343191475375e-05,
+      "learning_rate": 5.615683973893234e-07,
+      "loss": 0.08670675754547119,
+      "num_tokens": 5244470.0,
+      "reward": 0.4252437949180603,
+      "reward_std": 0.2996494472026825,
+      "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+      "rewards/true_env_reward_fn/std": 0.2996494472026825,
       "step": 212,
-      "step_time": 3.415528882000217
+      "step_time": 18.68646409699977
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5952,26 +5952,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 59.875,
-      "completions/mean_terminated_length": 59.875,
+      "completions/max_length": 133.0,
+      "completions/max_terminated_length": 133.0,
+      "completions/mean_length": 72.39583587646484,
+      "completions/mean_terminated_length": 72.39583587646484,
       "completions/min_length": 32.0,
       "completions/min_terminated_length": 32.0,
-      "entropy": 1.185910701751709,
-      "epoch": 1.7317073170731707,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001214127623825334,
-      "kl": 1.3199866316426778e-05,
-      "learning_rate": 3.659757011662538e-07,
-      "loss": 6.776077725589857e-07,
-      "num_tokens": 880344.0,
-      "reward": 0.7329437732696533,
-      "reward_std": 0.22123214602470398,
-      "rewards/true_env_reward_fn/mean": 0.7329437732696533,
-      "rewards/true_env_reward_fn/std": 0.22123214602470398,
+      "entropy": 1.3110275566577911,
+      "epoch": 5.195121951219512,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06817487627267838,
+      "kl": 0.00010393545744591393,
+      "learning_rate": 5.573417126992002e-07,
+      "loss": 0.04062439873814583,
+      "num_tokens": 5271369.0,
+      "reward": 0.4340522885322571,
+      "reward_std": 0.26940545439720154,
+      "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+      "rewards/true_env_reward_fn/std": 0.26940542459487915,
       "step": 213,
-      "step_time": 3.5156538789997285
+      "step_time": 11.3590317649996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -5980,26 +5980,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 162.0,
-      "completions/max_terminated_length": 162.0,
-      "completions/mean_length": 90.75,
-      "completions/mean_terminated_length": 90.75,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.2697569727897644,
-      "epoch": 1.7398373983739837,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 54.4375,
+      "completions/mean_terminated_length": 54.4375,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.2018881738185883,
+      "epoch": 5.219512195121951,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1553589552640915,
-      "kl": 1.621047795197228e-05,
-      "learning_rate": 3.622059472447875e-07,
-      "loss": -0.111361563205719,
-      "num_tokens": 885006.0,
-      "reward": 0.5490846633911133,
-      "reward_std": 0.14710450172424316,
-      "rewards/true_env_reward_fn/mean": 0.5490846633911133,
-      "rewards/true_env_reward_fn/std": 0.14710448682308197,
+      "grad_norm": 0.10059589147567749,
+      "kl": 0.00011373830420779996,
+      "learning_rate": 5.531108716309547e-07,
+      "loss": -0.09816272556781769,
+      "num_tokens": 5288638.0,
+      "reward": 0.5691710710525513,
+      "reward_std": 0.25253745913505554,
+      "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+      "rewards/true_env_reward_fn/std": 0.25253745913505554,
       "step": 214,
-      "step_time": 7.051423932000034
+      "step_time": 9.633293675000004
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6008,26 +6008,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 51.625,
-      "completions/mean_terminated_length": 51.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.1904898285865784,
-      "epoch": 1.7479674796747968,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19009968638420105,
-      "kl": 1.6167180092452327e-05,
-      "learning_rate": 3.5843957742810864e-07,
-      "loss": -0.006048411130905151,
-      "num_tokens": 888255.0,
-      "reward": 0.5893601179122925,
-      "reward_std": 0.16021940112113953,
-      "rewards/true_env_reward_fn/mean": 0.5893601179122925,
-      "rewards/true_env_reward_fn/std": 0.16021938621997833,
+      "completions/max_length": 157.0,
+      "completions/max_terminated_length": 157.0,
+      "completions/mean_length": 67.22917175292969,
+      "completions/mean_terminated_length": 67.22917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.1958912312984467,
+      "epoch": 5.2439024390243905,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0779944583773613,
+      "kl": 7.025236300250981e-05,
+      "learning_rate": 5.488761808544509e-07,
+      "loss": 0.03371567651629448,
+      "num_tokens": 5312081.0,
+      "reward": 0.44519662857055664,
+      "reward_std": 0.22201679646968842,
+      "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+      "rewards/true_env_reward_fn/std": 0.22201678156852722,
       "step": 215,
-      "step_time": 3.0270869319992926
+      "step_time": 12.463993090999338
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6036,26 +6036,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 55.875,
-      "completions/mean_terminated_length": 55.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.2750649452209473,
-      "epoch": 1.7560975609756098,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1521027386188507,
-      "kl": 4.317680577514693e-05,
-      "learning_rate": 3.5467692895949205e-07,
-      "loss": -0.04247616231441498,
-      "num_tokens": 892490.0,
-      "reward": 0.1841849982738495,
-      "reward_std": 0.30015870928764343,
-      "rewards/true_env_reward_fn/mean": 0.1841849982738495,
-      "rewards/true_env_reward_fn/std": 0.30015870928764343,
+      "completions/max_length": 234.0,
+      "completions/max_terminated_length": 234.0,
+      "completions/mean_length": 68.27083587646484,
+      "completions/mean_terminated_length": 68.27083587646484,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 1.2146256864070892,
+      "epoch": 5.2682926829268295,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06834157556295395,
+      "kl": 0.00010125362859980669,
+      "learning_rate": 5.446379473185971e-07,
+      "loss": -0.02198379673063755,
+      "num_tokens": 5334238.0,
+      "reward": 0.5273472666740417,
+      "reward_std": 0.21954773366451263,
+      "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+      "rewards/true_env_reward_fn/std": 0.21954771876335144,
       "step": 216,
-      "step_time": 3.461749838998003
+      "step_time": 15.126136884000061
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6064,26 +6064,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 62.25,
-      "completions/mean_terminated_length": 62.25,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.2730335593223572,
-      "epoch": 1.7642276422764227,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11831019073724747,
-      "kl": 2.1095927877468057e-05,
-      "learning_rate": 3.509183387490009e-07,
-      "loss": -0.051231447607278824,
-      "num_tokens": 894688.0,
-      "reward": 0.49520131945610046,
-      "reward_std": 0.3804744482040405,
-      "rewards/true_env_reward_fn/mean": 0.49520131945610046,
-      "rewards/true_env_reward_fn/std": 0.38047441840171814,
+      "completions/max_length": 103.0,
+      "completions/max_terminated_length": 103.0,
+      "completions/mean_length": 61.333335876464844,
+      "completions/mean_terminated_length": 61.333335876464844,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2872387170791626,
+      "epoch": 5.2926829268292686,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09578828513622284,
+      "kl": 7.404103780572768e-05,
+      "learning_rate": 5.403964782290961e-07,
+      "loss": -0.041091397404670715,
+      "num_tokens": 5351798.0,
+      "reward": 0.5464547872543335,
+      "reward_std": 0.16224616765975952,
+      "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+      "rewards/true_env_reward_fn/std": 0.16224615275859833,
       "step": 217,
-      "step_time": 3.7184635590019752
+      "step_time": 7.2901647220001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6092,26 +6092,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 47.75,
-      "completions/mean_terminated_length": 47.75,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.3015827536582947,
-      "epoch": 1.7723577235772359,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21724702417850494,
-      "kl": 3.562447545846226e-05,
-      "learning_rate": 3.471641433433191e-07,
-      "loss": -0.028775859624147415,
-      "num_tokens": 899066.0,
-      "reward": 0.3376166820526123,
-      "reward_std": 0.21976198256015778,
-      "rewards/true_env_reward_fn/mean": 0.3376166820526123,
-      "rewards/true_env_reward_fn/std": 0.21976199746131897,
+      "completions/max_length": 186.0,
+      "completions/max_terminated_length": 186.0,
+      "completions/mean_length": 84.89583587646484,
+      "completions/mean_terminated_length": 84.89583587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.4322342276573181,
+      "epoch": 5.317073170731708,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0667513981461525,
+      "kl": 2.497344303264981e-05,
+      "learning_rate": 5.361520810261778e-07,
+      "loss": -0.06392769515514374,
+      "num_tokens": 5375369.0,
+      "reward": 0.5213420391082764,
+      "reward_std": 0.1436246931552887,
+      "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+      "rewards/true_env_reward_fn/std": 0.1436246782541275,
       "step": 218,
-      "step_time": 3.344433074000335
+      "step_time": 12.217936152999755
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6120,26 +6120,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 53.125,
-      "completions/mean_terminated_length": 53.125,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.1390373706817627,
-      "epoch": 1.7804878048780488,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001077545020962134,
-      "kl": 1.554161144667887e-05,
-      "learning_rate": 3.434146788956166e-07,
-      "loss": 7.676237032683275e-07,
-      "num_tokens": 902263.0,
-      "reward": 0.5682899951934814,
-      "reward_std": 0.1217179074883461,
-      "rewards/true_env_reward_fn/mean": 0.5682899951934814,
-      "rewards/true_env_reward_fn/std": 0.1217179074883461,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 72.39583587646484,
+      "completions/mean_terminated_length": 72.39583587646484,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2382279634475708,
+      "epoch": 5.341463414634147,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07372384518384933,
+      "kl": 9.740726909512887e-05,
+      "learning_rate": 5.319050633623141e-07,
+      "loss": 0.010912742465734482,
+      "num_tokens": 5397956.0,
+      "reward": 0.5623860359191895,
+      "reward_std": 0.285375714302063,
+      "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+      "rewards/true_env_reward_fn/std": 0.285375714302063,
       "step": 219,
-      "step_time": 3.0424018219982827
+      "step_time": 13.449634822999997
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6148,26 +6148,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 61.875,
-      "completions/mean_terminated_length": 61.875,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.1991714239120483,
-      "epoch": 1.7886178861788617,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11428863555192947,
-      "kl": 1.1265870853094384e-05,
-      "learning_rate": 3.3967028113545045e-07,
-      "loss": -0.02927359938621521,
-      "num_tokens": 906206.0,
-      "reward": 0.4925253391265869,
-      "reward_std": 0.10813789069652557,
-      "rewards/true_env_reward_fn/mean": 0.4925253391265869,
-      "rewards/true_env_reward_fn/std": 0.10813789069652557,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 65.39583587646484,
+      "completions/mean_terminated_length": 65.39583587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2669419348239899,
+      "epoch": 5.365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07886125147342682,
+      "kl": 0.00011165817522851285,
+      "learning_rate": 5.276557330799203e-07,
+      "loss": -0.02433733269572258,
+      "num_tokens": 5418511.0,
+      "reward": 0.6075118780136108,
+      "reward_std": 0.30748677253723145,
+      "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+      "rewards/true_env_reward_fn/std": 0.30748677253723145,
       "step": 220,
-      "step_time": 3.5531271640011255
+      "step_time": 12.051496982000117
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6176,26 +6176,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 55.875,
-      "completions/mean_terminated_length": 55.875,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2546668648719788,
-      "epoch": 1.796747967479675,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1504630595445633,
-      "kl": 1.5663241811125772e-05,
-      "learning_rate": 3.3593128533870314e-07,
-      "loss": 0.04806854575872421,
-      "num_tokens": 909149.0,
-      "reward": 0.49599751830101013,
-      "reward_std": 0.15429075062274933,
-      "rewards/true_env_reward_fn/mean": 0.49599751830101013,
-      "rewards/true_env_reward_fn/std": 0.15429075062274933,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 66.0625,
+      "completions/mean_terminated_length": 66.0625,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2560602128505707,
+      "epoch": 5.390243902439025,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08607357740402222,
+      "kl": 8.424731549894204e-05,
+      "learning_rate": 5.234043981890393e-07,
+      "loss": 0.013634156435728073,
+      "num_tokens": 5439690.0,
+      "reward": 0.5080039501190186,
+      "reward_std": 0.21975299715995789,
+      "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+      "rewards/true_env_reward_fn/std": 0.21975299715995789,
       "step": 221,
-      "step_time": 3.1345955030010373
+      "step_time": 10.435893627000496
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6204,26 +6204,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 58.25,
-      "completions/mean_terminated_length": 58.25,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.1428714394569397,
-      "epoch": 1.8048780487804879,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.310600969707593e-05,
-      "kl": 1.1706195891747484e-05,
-      "learning_rate": 3.321980262975613e-07,
-      "loss": 5.957842290627013e-07,
-      "num_tokens": 914211.0,
-      "reward": 0.3258306384086609,
-      "reward_std": 0.4338511824607849,
-      "rewards/true_env_reward_fn/mean": 0.3258306384086609,
-      "rewards/true_env_reward_fn/std": 0.4338512122631073,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 70.95833587646484,
+      "completions/mean_terminated_length": 70.95833587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3081265985965729,
+      "epoch": 5.414634146341464,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0876006931066513,
+      "kl": 9.127605972025776e-05,
+      "learning_rate": 5.191513668450177e-07,
+      "loss": -0.01968565583229065,
+      "num_tokens": 5468216.0,
+      "reward": 0.3679848611354828,
+      "reward_std": 0.237859845161438,
+      "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+      "rewards/true_env_reward_fn/std": 0.237859845161438,
       "step": 222,
-      "step_time": 3.8445859539988305
+      "step_time": 11.524256381999294
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6232,26 +6232,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 55.875,
-      "completions/mean_terminated_length": 55.875,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.1629019975662231,
-      "epoch": 1.8130081300813008,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010947151895379648,
-      "kl": 1.3530024261854123e-05,
-      "learning_rate": 3.2847083829053923e-07,
-      "loss": 6.723923888785066e-07,
-      "num_tokens": 916850.0,
-      "reward": 0.5956059694290161,
-      "reward_std": 0.09251586347818375,
-      "rewards/true_env_reward_fn/mean": 0.5956059694290161,
-      "rewards/true_env_reward_fn/std": 0.09251587092876434,
+      "completions/max_length": 158.0,
+      "completions/max_terminated_length": 158.0,
+      "completions/mean_length": 67.66667175292969,
+      "completions/mean_terminated_length": 67.66667175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2990808486938477,
+      "epoch": 5.439024390243903,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07685395330190659,
+      "kl": 0.00014209141045284923,
+      "learning_rate": 5.148969473261679e-07,
+      "loss": -0.01107364147901535,
+      "num_tokens": 5488040.0,
+      "reward": 0.5435788035392761,
+      "reward_std": 0.35419100522994995,
+      "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+      "rewards/true_env_reward_fn/std": 0.35419100522994995,
       "step": 223,
-      "step_time": 3.3021794950000185
+      "step_time": 11.945272217000365
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6260,26 +6260,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 61.375,
-      "completions/mean_terminated_length": 61.375,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.1237311363220215,
-      "epoch": 1.821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14958374202251434,
-      "kl": 4.083753810846247e-05,
-      "learning_rate": 3.2475005505254657e-07,
-      "loss": 0.09789139032363892,
-      "num_tokens": 919901.0,
-      "reward": 0.4179220199584961,
-      "reward_std": 0.2486819326877594,
-      "rewards/true_env_reward_fn/mean": 0.4179220199584961,
-      "rewards/true_env_reward_fn/std": 0.2486819475889206,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 65.4375,
+      "completions/mean_terminated_length": 65.4375,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.284348964691162,
+      "epoch": 5.463414634146342,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.058678388595581055,
+      "kl": 3.3754420201148605e-05,
+      "learning_rate": 5.106414480114238e-07,
+      "loss": -0.01714605651795864,
+      "num_tokens": 5513005.0,
+      "reward": 0.43836766481399536,
+      "reward_std": 0.26744046807289124,
+      "rewards/true_env_reward_fn/mean": 0.438367635011673,
+      "rewards/true_env_reward_fn/std": 0.26744046807289124,
       "step": 224,
-      "step_time": 4.954825423001239
+      "step_time": 10.563708176000091
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6288,26 +6288,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 52.375,
-      "completions/mean_terminated_length": 52.375,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.1394256949424744,
-      "epoch": 1.8292682926829267,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1480269879102707,
-      "kl": 3.2915593692450784e-05,
-      "learning_rate": 3.210360097450052e-07,
-      "loss": 0.11319300532341003,
-      "num_tokens": 923840.0,
-      "reward": 0.4335233271121979,
-      "reward_std": 0.31114333868026733,
-      "rewards/true_env_reward_fn/mean": 0.4335233271121979,
-      "rewards/true_env_reward_fn/std": 0.31114333868026733,
+      "completions/max_length": 170.0,
+      "completions/max_terminated_length": 170.0,
+      "completions/mean_length": 65.875,
+      "completions/mean_terminated_length": 65.875,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.3357127904891968,
+      "epoch": 5.487804878048781,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07269197702407837,
+      "kl": 3.727909597728285e-05,
+      "learning_rate": 5.063851773579869e-07,
+      "loss": -0.01076439768075943,
+      "num_tokens": 5530655.0,
+      "reward": 0.6185358762741089,
+      "reward_std": 0.19721543788909912,
+      "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+      "rewards/true_env_reward_fn/std": 0.19721543788909912,
       "step": 225,
-      "step_time": 3.777259659000265
+      "step_time": 10.372150705000422
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6316,26 +6316,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 64.25,
-      "completions/mean_terminated_length": 64.25,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.354669451713562,
-      "epoch": 1.8373983739837398,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13672730326652527,
-      "kl": 2.2323275516100693e-05,
-      "learning_rate": 3.173290349260188e-07,
-      "loss": -0.0521523654460907,
-      "num_tokens": 927882.0,
-      "reward": 0.57341468334198,
-      "reward_std": 0.11576741933822632,
-      "rewards/true_env_reward_fn/mean": 0.57341468334198,
-      "rewards/true_env_reward_fn/std": 0.11576744168996811,
+      "completions/max_length": 151.0,
+      "completions/max_terminated_length": 151.0,
+      "completions/mean_length": 64.35417175292969,
+      "completions/mean_terminated_length": 64.35417175292969,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2975924611091614,
+      "epoch": 5.512195121951219,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0875314474105835,
+      "kl": 9.186910938296933e-05,
+      "learning_rate": 5.021284438789693e-07,
+      "loss": 0.048179637640714645,
+      "num_tokens": 5554032.0,
+      "reward": 0.45552024245262146,
+      "reward_std": 0.31553781032562256,
+      "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+      "rewards/true_env_reward_fn/std": 0.31553778052330017,
       "step": 226,
-      "step_time": 3.613498073998926
+      "step_time": 12.18860100899974
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6344,26 +6344,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 64.875,
-      "completions/mean_terminated_length": 64.875,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.1750767230987549,
-      "epoch": 1.845528455284553,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.230674147605896,
-      "kl": 1.800864629331045e-05,
-      "learning_rate": 3.136294625205945e-07,
-      "loss": 0.058730173856019974,
-      "num_tokens": 930225.0,
-      "reward": 0.6635305881500244,
-      "reward_std": 0.1901332288980484,
-      "rewards/true_env_reward_fn/mean": 0.6635305881500244,
-      "rewards/true_env_reward_fn/std": 0.1901332437992096,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 63.208335876464844,
+      "completions/mean_terminated_length": 63.208335876464844,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2268341779708862,
+      "epoch": 5.536585365853659,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08318481594324112,
+      "kl": 8.052505017985823e-05,
+      "learning_rate": 4.978715561210307e-07,
+      "loss": 0.027963606640696526,
+      "num_tokens": 5582898.0,
+      "reward": 0.36310288310050964,
+      "reward_std": 0.3131144344806671,
+      "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+      "rewards/true_env_reward_fn/std": 0.3131144344806671,
       "step": 227,
-      "step_time": 3.3819083769976714
+      "step_time": 12.510411257000214
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6372,26 +6372,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 55.0,
-      "completions/max_terminated_length": 55.0,
-      "completions/mean_length": 40.875,
-      "completions/mean_terminated_length": 40.875,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.2825847864151,
-      "epoch": 1.8536585365853657,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.30391862988471985,
-      "kl": 2.0106223928451072e-05,
-      "learning_rate": 3.0993762379092235e-07,
-      "loss": -0.10232458263635635,
-      "num_tokens": 933048.0,
-      "reward": 0.45667415857315063,
-      "reward_std": 0.26290765404701233,
-      "rewards/true_env_reward_fn/mean": 0.45667415857315063,
-      "rewards/true_env_reward_fn/std": 0.26290765404701233,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 64.9375,
+      "completions/mean_terminated_length": 64.9375,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.1668500006198883,
+      "epoch": 5.560975609756097,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0727877989411354,
+      "kl": 7.824771091691218e-05,
+      "learning_rate": 4.936148226420132e-07,
+      "loss": -0.01629078947007656,
+      "num_tokens": 5604791.0,
+      "reward": 0.5653349161148071,
+      "reward_std": 0.2849632799625397,
+      "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+      "rewards/true_env_reward_fn/std": 0.2849632799625397,
       "step": 228,
-      "step_time": 2.698590726000475
+      "step_time": 10.083805716999905
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6400,26 +6400,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 55.0,
-      "completions/max_terminated_length": 55.0,
-      "completions/mean_length": 41.125,
-      "completions/mean_terminated_length": 41.125,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 1.3104369640350342,
-      "epoch": 1.8617886178861789,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.28224605321884155,
-      "kl": 6.227439189387951e-05,
-      "learning_rate": 3.06253849306714e-07,
-      "loss": -0.008679002523422241,
-      "num_tokens": 938217.0,
-      "reward": 0.2567799985408783,
-      "reward_std": 0.2555168867111206,
-      "rewards/true_env_reward_fn/mean": 0.2567799985408783,
-      "rewards/true_env_reward_fn/std": 0.255516916513443,
+      "completions/max_length": 133.0,
+      "completions/max_terminated_length": 133.0,
+      "completions/mean_length": 68.29167175292969,
+      "completions/mean_terminated_length": 68.29167175292969,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.307439923286438,
+      "epoch": 5.585365853658536,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06797961890697479,
+      "kl": 6.601300174224889e-05,
+      "learning_rate": 4.893585519885763e-07,
+      "loss": -0.036383360624313354,
+      "num_tokens": 5631629.0,
+      "reward": 0.49251794815063477,
+      "reward_std": 0.22127023339271545,
+      "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+      "rewards/true_env_reward_fn/std": 0.22127023339271545,
       "step": 229,
-      "step_time": 2.7451177700022527
+      "step_time": 13.650024606999523
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6428,26 +6428,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 61.0,
-      "completions/max_terminated_length": 61.0,
-      "completions/mean_length": 49.0,
-      "completions/mean_terminated_length": 49.0,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0164751410484314,
-      "epoch": 1.8699186991869918,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011273886048002169,
-      "kl": 1.1985231139988173e-05,
-      "learning_rate": 3.0257846891560323e-07,
-      "loss": 5.987301960885816e-07,
-      "num_tokens": 941049.0,
-      "reward": 0.6971603631973267,
-      "reward_std": 0.2594861686229706,
-      "rewards/true_env_reward_fn/mean": 0.6971603631973267,
-      "rewards/true_env_reward_fn/std": 0.2594861686229706,
+      "completions/max_length": 194.0,
+      "completions/max_terminated_length": 194.0,
+      "completions/mean_length": 63.395835876464844,
+      "completions/mean_terminated_length": 63.395835876464844,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.3524385392665863,
+      "epoch": 5.609756097560975,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.12484809756278992,
+      "kl": 0.0002558713749749586,
+      "learning_rate": 4.85103052673832e-07,
+      "loss": -0.0045075975358486176,
+      "num_tokens": 5666712.0,
+      "reward": 0.2605232000350952,
+      "reward_std": 0.22926004230976105,
+      "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+      "rewards/true_env_reward_fn/std": 0.22926005721092224,
       "step": 230,
-      "step_time": 2.917641182999432
+      "step_time": 15.322059910999542
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6456,26 +6456,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 13.0,
-      "completions/min_terminated_length": 13.0,
-      "entropy": 1.1991845965385437,
-      "epoch": 1.8780487804878048,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00015363915008492768,
-      "kl": 1.4284144981502322e-05,
-      "learning_rate": 2.989118117136107e-07,
-      "loss": 6.798551339670666e-07,
-      "num_tokens": 945403.0,
-      "reward": 0.31745320558547974,
-      "reward_std": 0.42489534616470337,
-      "rewards/true_env_reward_fn/mean": 0.31745320558547974,
-      "rewards/true_env_reward_fn/std": 0.42489534616470337,
+      "completions/max_length": 136.0,
+      "completions/max_terminated_length": 136.0,
+      "completions/mean_length": 68.08333587646484,
+      "completions/mean_terminated_length": 68.08333587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2202666401863098,
+      "epoch": 5.634146341463414,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07976125925779343,
+      "kl": 8.684267686476232e-05,
+      "learning_rate": 4.808486331549823e-07,
+      "loss": 0.012383833527565002,
+      "num_tokens": 5702004.0,
+      "reward": 0.20571085810661316,
+      "reward_std": 0.2265808880329132,
+      "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+      "rewards/true_env_reward_fn/std": 0.2265808880329132,
       "step": 231,
-      "step_time": 5.5585464220002905
+      "step_time": 14.180213787999492
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6484,26 +6484,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 71.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 60.375,
-      "completions/mean_terminated_length": 60.375,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2020843029022217,
-      "epoch": 1.886178861788618,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13684362173080444,
-      "kl": 3.651866154541494e-05,
-      "learning_rate": 2.952542060156773e-07,
-      "loss": 0.025869816541671753,
-      "num_tokens": 949126.0,
-      "reward": 0.3984018564224243,
-      "reward_std": 0.4202974736690521,
-      "rewards/true_env_reward_fn/mean": 0.3984018564224243,
-      "rewards/true_env_reward_fn/std": 0.4202974736690521,
+      "completions/max_length": 312.0,
+      "completions/max_terminated_length": 312.0,
+      "completions/mean_length": 78.83333587646484,
+      "completions/mean_terminated_length": 78.83333587646484,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "entropy": 1.3096380531787872,
+      "epoch": 5.658536585365853,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08384906500577927,
+      "kl": 8.733692084206268e-05,
+      "learning_rate": 4.7659560181096067e-07,
+      "loss": 0.02098608762025833,
+      "num_tokens": 5729780.0,
+      "reward": 0.4599723219871521,
+      "reward_std": 0.21680118143558502,
+      "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+      "rewards/true_env_reward_fn/std": 0.21680118143558502,
       "step": 232,
-      "step_time": 3.4383463869999105
+      "step_time": 21.68401394800003
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6512,26 +6512,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 58.625,
-      "completions/mean_terminated_length": 58.625,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2224581837654114,
-      "epoch": 1.8943089430894309,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20249564945697784,
-      "kl": 3.106597978330683e-05,
-      "learning_rate": 2.9160597932626605e-07,
-      "loss": -0.003915777429938316,
-      "num_tokens": 952815.0,
-      "reward": 0.4141089916229248,
-      "reward_std": 0.3064958453178406,
-      "rewards/true_env_reward_fn/mean": 0.4141089916229248,
-      "rewards/true_env_reward_fn/std": 0.3064958453178406,
+      "completions/max_length": 83.0,
+      "completions/max_terminated_length": 83.0,
+      "completions/mean_length": 53.520835876464844,
+      "completions/mean_terminated_length": 53.520835876464844,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.2158988416194916,
+      "epoch": 5.682926829268292,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08225176483392715,
+      "kl": 9.274652438762132e-05,
+      "learning_rate": 4.7234426692007977e-07,
+      "loss": -0.034079790115356445,
+      "num_tokens": 5753397.0,
+      "reward": 0.5273070335388184,
+      "reward_std": 0.2194610834121704,
+      "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+      "rewards/true_env_reward_fn/std": 0.21946106851100922,
       "step": 233,
-      "step_time": 3.6410487339999236
+      "step_time": 7.915307780000603
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6540,26 +6540,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 61.625,
-      "completions/mean_terminated_length": 61.625,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.1212781071662903,
-      "epoch": 1.9024390243902438,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14502985775470734,
-      "kl": 3.0683338081871625e-05,
-      "learning_rate": 2.879674583100372e-07,
-      "loss": -0.0860406681895256,
-      "num_tokens": 958656.0,
-      "reward": 0.30326664447784424,
-      "reward_std": 0.24800051748752594,
-      "rewards/true_env_reward_fn/mean": 0.30326664447784424,
-      "rewards/true_env_reward_fn/std": 0.24800053238868713,
+      "completions/max_length": 138.0,
+      "completions/max_terminated_length": 138.0,
+      "completions/mean_length": 78.0,
+      "completions/mean_terminated_length": 78.0,
+      "completions/min_length": 53.0,
+      "completions/min_terminated_length": 53.0,
+      "entropy": 1.2362721860408783,
+      "epoch": 5.7073170731707314,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06290840357542038,
+      "kl": 7.484563047910342e-05,
+      "learning_rate": 4.6809493663768575e-07,
+      "loss": -0.008873865008354187,
+      "num_tokens": 5778869.0,
+      "reward": 0.5283026695251465,
+      "reward_std": 0.195759579539299,
+      "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+      "rewards/true_env_reward_fn/std": 0.195759579539299,
       "step": 234,
-      "step_time": 3.8798253620007017
+      "step_time": 10.278297286999987
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6568,26 +6568,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 60.75,
-      "completions/mean_terminated_length": 60.75,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.425286054611206,
-      "epoch": 1.910569105691057,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2035023272037506,
-      "kl": 5.512987627298571e-05,
-      "learning_rate": 2.843389687625986e-07,
-      "loss": -0.010439477860927582,
-      "num_tokens": 962638.0,
-      "reward": 0.3157375156879425,
-      "reward_std": 0.46047845482826233,
-      "rewards/true_env_reward_fn/mean": 0.3157375156879425,
-      "rewards/true_env_reward_fn/std": 0.46047845482826233,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 64.47917175292969,
+      "completions/mean_terminated_length": 64.47917175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3364675641059875,
+      "epoch": 5.7317073170731705,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09685558825731277,
+      "kl": 0.00013002969353692606,
+      "learning_rate": 4.638479189738224e-07,
+      "loss": 0.05070740357041359,
+      "num_tokens": 5804436.0,
+      "reward": 0.44921523332595825,
+      "reward_std": 0.22693434357643127,
+      "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+      "rewards/true_env_reward_fn/std": 0.22693434357643127,
       "step": 235,
-      "step_time": 4.114513064998391
+      "step_time": 10.754199091999908
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6596,26 +6596,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 120.0,
-      "completions/max_terminated_length": 120.0,
-      "completions/mean_length": 77.625,
-      "completions/mean_terminated_length": 77.625,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.4385854601860046,
-      "epoch": 1.91869918699187,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12982772290706635,
-      "kl": 1.2591926861205138e-05,
-      "learning_rate": 2.807208355813339e-07,
-      "loss": 0.09093751758337021,
-      "num_tokens": 965755.0,
-      "reward": 0.6140732765197754,
-      "reward_std": 0.27462607622146606,
-      "rewards/true_env_reward_fn/mean": 0.6140732765197754,
-      "rewards/true_env_reward_fn/std": 0.27462607622146606,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 62.0625,
+      "completions/mean_terminated_length": 62.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2267529368400574,
+      "epoch": 5.7560975609756095,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.10451620817184448,
+      "kl": 6.754635069228243e-05,
+      "learning_rate": 4.596035217709039e-07,
+      "loss": -0.02925686165690422,
+      "num_tokens": 5826895.0,
+      "reward": 0.49523353576660156,
+      "reward_std": 0.15093794465065002,
+      "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+      "rewards/true_env_reward_fn/std": 0.15093792974948883,
       "step": 236,
-      "step_time": 5.147667763001664
+      "step_time": 9.666070583999954
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6624,26 +6624,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 100.0,
-      "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 71.375,
-      "completions/mean_terminated_length": 71.375,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.3476852178573608,
-      "epoch": 1.9268292682926829,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14480431377887726,
-      "kl": 3.8014684832887724e-05,
-      "learning_rate": 2.771133827363101e-07,
-      "loss": -0.07322391867637634,
-      "num_tokens": 970918.0,
-      "reward": 0.3427826166152954,
-      "reward_std": 0.42430612444877625,
-      "rewards/true_env_reward_fn/mean": 0.3427826166152954,
-      "rewards/true_env_reward_fn/std": 0.42430609464645386,
+      "completions/max_length": 144.0,
+      "completions/max_terminated_length": 144.0,
+      "completions/mean_length": 67.79167175292969,
+      "completions/mean_terminated_length": 67.79167175292969,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2110519707202911,
+      "epoch": 5.780487804878049,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07890205830335617,
+      "kl": 0.0001245876064785989,
+      "learning_rate": 4.5536205268140286e-07,
+      "loss": -0.03693925589323044,
+      "num_tokens": 5859973.0,
+      "reward": 0.2979053258895874,
+      "reward_std": 0.23015481233596802,
+      "rewards/true_env_reward_fn/mean": 0.297905296087265,
+      "rewards/true_env_reward_fn/std": 0.2301548272371292,
       "step": 237,
-      "step_time": 4.67846887900123
+      "step_time": 13.971699990999241
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6652,26 +6652,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 221.0,
-      "completions/max_terminated_length": 221.0,
-      "completions/mean_length": 94.25,
-      "completions/mean_terminated_length": 94.25,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.3279914855957031,
-      "epoch": 1.934959349593496,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10826019197702408,
-      "kl": 1.3074863090878353e-05,
-      "learning_rate": 2.7351693324127037e-07,
-      "loss": -0.05556309223175049,
-      "num_tokens": 975120.0,
-      "reward": 0.64573073387146,
-      "reward_std": 0.22739914059638977,
-      "rewards/true_env_reward_fn/mean": 0.64573073387146,
-      "rewards/true_env_reward_fn/std": 0.22739915549755096,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 65.52083587646484,
+      "completions/mean_terminated_length": 65.52083587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2291057407855988,
+      "epoch": 5.804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09152546525001526,
+      "kl": 0.00013101351214572787,
+      "learning_rate": 4.511238191455491e-07,
+      "loss": 0.043641336262226105,
+      "num_tokens": 5886118.0,
+      "reward": 0.41334670782089233,
+      "reward_std": 0.19480590522289276,
+      "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+      "rewards/true_env_reward_fn/std": 0.19480590522289276,
       "step": 238,
-      "step_time": 9.220254810001279
+      "step_time": 10.805698846999803
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6680,26 +6680,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 66.75,
-      "completions/mean_terminated_length": 66.75,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.3695034384727478,
-      "epoch": 1.943089430894309,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12166028469800949,
-      "kl": 2.6563114261080045e-05,
-      "learning_rate": 2.6993180912471055e-07,
-      "loss": -0.05334407091140747,
-      "num_tokens": 980254.0,
-      "reward": 0.37203267216682434,
-      "reward_std": 0.20089927315711975,
-      "rewards/true_env_reward_fn/mean": 0.37203267216682434,
-      "rewards/true_env_reward_fn/std": 0.20089928805828094,
+      "completions/max_length": 120.0,
+      "completions/max_terminated_length": 120.0,
+      "completions/mean_length": 58.833335876464844,
+      "completions/mean_terminated_length": 58.833335876464844,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.3134913444519043,
+      "epoch": 5.829268292682927,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07343924045562744,
+      "kl": 0.00019655993492051493,
+      "learning_rate": 4.4688912836904533e-07,
+      "loss": 0.015309082344174385,
+      "num_tokens": 5908334.0,
+      "reward": 0.4438478946685791,
+      "reward_std": 0.27188754081726074,
+      "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+      "rewards/true_env_reward_fn/std": 0.27188754081726074,
       "step": 239,
-      "step_time": 4.224964968001586
+      "step_time": 9.688736522
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6708,26 +6708,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 53.75,
-      "completions/mean_terminated_length": 53.75,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.247464120388031,
-      "epoch": 1.951219512195122,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2346416860818863,
-      "kl": 5.1520751185307745e-05,
-      "learning_rate": 2.6635833140104405e-07,
-      "loss": -0.0905834436416626,
-      "num_tokens": 983244.0,
-      "reward": 0.575507640838623,
-      "reward_std": 0.17391785979270935,
-      "rewards/true_env_reward_fn/mean": 0.575507640838623,
-      "rewards/true_env_reward_fn/std": 0.17391787469387054,
+      "completions/max_length": 464.0,
+      "completions/max_terminated_length": 464.0,
+      "completions/mean_length": 79.14583587646484,
+      "completions/mean_terminated_length": 79.14583587646484,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3148745000362396,
+      "epoch": 5.853658536585366,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06567981839179993,
+      "kl": 4.445325248525478e-05,
+      "learning_rate": 4.4265828730079977e-07,
+      "loss": 0.021218176931142807,
+      "num_tokens": 5929645.0,
+      "reward": 0.5866342186927795,
+      "reward_std": 0.13780196011066437,
+      "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+      "rewards/true_env_reward_fn/std": 0.13780196011066437,
       "step": 240,
-      "step_time": 3.357481237999309
+      "step_time": 27.046819901000163
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6736,26 +6736,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.2175387144088745,
-      "epoch": 1.959349593495935,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1870066374540329,
-      "kl": 2.2185965462995227e-05,
-      "learning_rate": 2.6279682004185894e-07,
-      "loss": -0.07365687191486359,
-      "num_tokens": 985574.0,
-      "reward": 0.6635305881500244,
-      "reward_std": 0.1901332288980484,
-      "rewards/true_env_reward_fn/mean": 0.6635305881500244,
-      "rewards/true_env_reward_fn/std": 0.1901332437992096,
+      "completions/max_length": 364.0,
+      "completions/max_terminated_length": 364.0,
+      "completions/mean_length": 70.33333587646484,
+      "completions/mean_terminated_length": 70.33333587646484,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 1.2104995846748352,
+      "epoch": 5.878048780487805,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.042626503854990005,
+      "kl": 6.413207393052289e-05,
+      "learning_rate": 4.3843160261067653e-07,
+      "loss": -0.0065308245830237865,
+      "num_tokens": 5950005.0,
+      "reward": 0.5593677759170532,
+      "reward_std": 0.23993276059627533,
+      "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+      "rewards/true_env_reward_fn/std": 0.23993274569511414,
       "step": 241,
-      "step_time": 4.527591582998866
+      "step_time": 22.238758486000734
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6764,26 +6764,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 98.0,
-      "completions/max_terminated_length": 98.0,
-      "completions/mean_length": 76.25,
-      "completions/mean_terminated_length": 76.25,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.2727615237236023,
-      "epoch": 1.967479674796748,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.226071804529056e-05,
-      "kl": 1.4014385214977665e-05,
-      "learning_rate": 2.592475939472668e-07,
-      "loss": 7.015369192231447e-07,
-      "num_tokens": 988868.0,
-      "reward": 0.7316612601280212,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.7316612601280212,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 83.0,
+      "completions/max_terminated_length": 83.0,
+      "completions/mean_length": 61.270835876464844,
+      "completions/mean_terminated_length": 61.270835876464844,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2132116258144379,
+      "epoch": 5.902439024390244,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05718924477696419,
+      "kl": 6.140609366411809e-05,
+      "learning_rate": 4.342093806672678e-07,
+      "loss": 0.012734346091747284,
+      "num_tokens": 5974626.0,
+      "reward": 0.4349059462547302,
+      "reward_std": 0.24915602803230286,
+      "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+      "rewards/true_env_reward_fn/std": 0.24915601313114166,
       "step": 242,
-      "step_time": 4.238274277000528
+      "step_time": 9.239750460000323
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6791,27 +6791,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 57.5,
-      "completions/mean_terminated_length": 57.5,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1519948840141296,
-      "epoch": 1.975609756097561,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12447996437549591,
-      "kl": 4.700180943473242e-05,
-      "learning_rate": 2.557109709173482e-07,
-      "loss": 0.08819369971752167,
-      "num_tokens": 994068.0,
-      "reward": 0.3696666657924652,
-      "reward_std": 0.18488828837871552,
-      "rewards/true_env_reward_fn/mean": 0.3696666657924652,
-      "rewards/true_env_reward_fn/std": 0.1848883032798767,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 96.0,
+      "completions/mean_length": 67.04167175292969,
+      "completions/mean_terminated_length": 57.574466705322266,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.7212344706058502,
+      "epoch": 5.926829268292683,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08872146159410477,
+      "kl": 0.00011613740025495645,
+      "learning_rate": 4.2999192751568557e-07,
+      "loss": -0.021363887935876846,
+      "num_tokens": 5997300.0,
+      "reward": 0.4436037540435791,
+      "reward_std": 0.28323379158973694,
+      "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+      "rewards/true_env_reward_fn/std": 0.28323376178741455,
       "step": 243,
-      "step_time": 3.3367313500002638
+      "step_time": 29.767976787999487
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6820,26 +6820,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 115.0,
-      "completions/max_terminated_length": 115.0,
-      "completions/mean_length": 68.25,
-      "completions/mean_terminated_length": 68.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.0537148416042328,
-      "epoch": 1.9837398373983741,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17681685090065002,
-      "kl": 3.6240851841284893e-05,
-      "learning_rate": 2.521872676236972e-07,
-      "loss": 0.05281040072441101,
-      "num_tokens": 1000650.0,
-      "reward": 0.12209999561309814,
-      "reward_std": 0.2502918243408203,
-      "rewards/true_env_reward_fn/mean": 0.12209999561309814,
-      "rewards/true_env_reward_fn/std": 0.2502918243408203,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 61.083335876464844,
+      "completions/mean_terminated_length": 61.083335876464844,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2353634238243103,
+      "epoch": 5.951219512195122,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07658001780509949,
+      "kl": 9.086773115996039e-05,
+      "learning_rate": 4.257795488553798e-07,
+      "loss": -0.005057391710579395,
+      "num_tokens": 6021752.0,
+      "reward": 0.49971556663513184,
+      "reward_std": 0.2643933594226837,
+      "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+      "rewards/true_env_reward_fn/std": 0.2643933594226837,
       "step": 244,
-      "step_time": 5.2957401019993995
+      "step_time": 9.764708648999658
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6847,27 +6847,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 140.0,
-      "completions/max_terminated_length": 140.0,
-      "completions/mean_length": 95.375,
-      "completions/mean_terminated_length": 95.375,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "entropy": 1.3126497864723206,
-      "epoch": 1.9918699186991868,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16036106646060944,
-      "kl": 6.061139720259234e-05,
-      "learning_rate": 2.48676799581066e-07,
-      "loss": 0.09418506920337677,
-      "num_tokens": 1005737.0,
-      "reward": 0.31175702810287476,
-      "reward_std": 0.38867074251174927,
-      "rewards/true_env_reward_fn/mean": 0.31175702810287476,
-      "rewards/true_env_reward_fn/std": 0.38867077231407166,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 216.0,
+      "completions/mean_length": 86.72917175292969,
+      "completions/mean_terminated_length": 77.68084716796875,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 1.2082330882549286,
+      "epoch": 5.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.059102799743413925,
+      "kl": 5.0710960749711376e-05,
+      "learning_rate": 4.215725500179787e-07,
+      "loss": 0.17436102032661438,
+      "num_tokens": 6049747.0,
+      "reward": 0.46306928992271423,
+      "reward_std": 0.26072925329208374,
+      "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+      "rewards/true_env_reward_fn/std": 0.26072925329208374,
       "step": 245,
-      "step_time": 6.259088058999623
+      "step_time": 31.997988874000384
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6876,26 +6876,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.0,
-      "completions/max_terminated_length": 64.0,
-      "completions/mean_length": 48.5,
-      "completions/mean_terminated_length": 48.5,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.21333646774292,
-      "epoch": 2.0,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23997871577739716,
-      "kl": 3.1378609492094256e-05,
-      "learning_rate": 2.4517988111911313e-07,
-      "loss": 0.010592922568321228,
-      "num_tokens": 1010869.0,
-      "reward": 0.33381664752960205,
-      "reward_std": 0.18213039636611938,
-      "rewards/true_env_reward_fn/mean": 0.33381664752960205,
-      "rewards/true_env_reward_fn/std": 0.18213039636611938,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 61.35416793823242,
+      "completions/mean_terminated_length": 61.35416793823242,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2454268038272858,
+      "epoch": 6.0,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07332625985145569,
+      "kl": 0.0001107546740968246,
+      "learning_rate": 4.1737123594515755e-07,
+      "loss": -0.013426866382360458,
+      "num_tokens": 6072668.0,
+      "reward": 0.45415109395980835,
+      "reward_std": 0.2937946617603302,
+      "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+      "rewards/true_env_reward_fn/std": 0.2937946617603302,
       "step": 246,
-      "step_time": 3.111915630997828
+      "step_time": 9.587768273000165
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6904,26 +6904,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 61.375,
-      "completions/mean_terminated_length": 61.375,
-      "completions/min_length": 58.0,
-      "completions/min_terminated_length": 58.0,
-      "entropy": 1.3962982892990112,
-      "epoch": 2.008130081300813,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21355785429477692,
-      "kl": 3.789625407080166e-05,
-      "learning_rate": 2.4169682535425927e-07,
-      "loss": 0.025682777166366577,
-      "num_tokens": 1014876.0,
-      "reward": 0.35749268531799316,
-      "reward_std": 0.29738906025886536,
-      "rewards/true_env_reward_fn/mean": 0.35749268531799316,
-      "rewards/true_env_reward_fn/std": 0.29738909006118774,
+      "completions/max_length": 137.0,
+      "completions/max_terminated_length": 137.0,
+      "completions/mean_length": 71.5625,
+      "completions/mean_terminated_length": 71.5625,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.1738699078559875,
+      "epoch": 6.024390243902439,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06093747541308403,
+      "kl": 7.893411338955048e-05,
+      "learning_rate": 4.131759111665348e-07,
+      "loss": 0.022837676107883453,
+      "num_tokens": 6095263.0,
+      "reward": 0.5188159346580505,
+      "reward_std": 0.3265886902809143,
+      "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+      "rewards/true_env_reward_fn/std": 0.3265886902809143,
       "step": 247,
-      "step_time": 3.35338095500083
+      "step_time": 14.232978527999876
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6932,26 +6932,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 73.875,
-      "completions/mean_terminated_length": 73.875,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.463137686252594,
-      "epoch": 2.016260162601626,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18188992142677307,
-      "kl": 1.746804719005013e-05,
-      "learning_rate": 2.382279441616492e-07,
-      "loss": -0.17857304215431213,
-      "num_tokens": 1018383.0,
-      "reward": 0.5329012274742126,
-      "reward_std": 0.055823445320129395,
-      "rewards/true_env_reward_fn/mean": 0.5329012274742126,
-      "rewards/true_env_reward_fn/std": 0.05582345277070999,
+      "completions/max_length": 186.0,
+      "completions/max_terminated_length": 186.0,
+      "completions/mean_length": 66.9375,
+      "completions/mean_terminated_length": 66.9375,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.3300949931144714,
+      "epoch": 6.048780487804878,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08098509907722473,
+      "kl": 0.00010828049380506855,
+      "learning_rate": 4.0898687977759887e-07,
+      "loss": 0.01631344109773636,
+      "num_tokens": 6125380.0,
+      "reward": 0.3338983952999115,
+      "reward_std": 0.19050820171833038,
+      "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+      "rewards/true_env_reward_fn/std": 0.19050820171833038,
       "step": 248,
-      "step_time": 5.210386754000865
+      "step_time": 13.248441182000079
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6960,26 +6960,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 211.0,
-      "completions/max_terminated_length": 211.0,
-      "completions/mean_length": 79.25,
-      "completions/mean_terminated_length": 79.25,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.4478936195373535,
-      "epoch": 2.024390243902439,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.744662434561178e-05,
-      "kl": 1.3336490155779757e-05,
-      "learning_rate": 2.3477354814722762e-07,
-      "loss": 6.725406365148956e-07,
-      "num_tokens": 1022753.0,
-      "reward": 0.5905972719192505,
-      "reward_std": 0.15080371499061584,
-      "rewards/true_env_reward_fn/mean": 0.5905972719192505,
-      "rewards/true_env_reward_fn/std": 0.15080371499061584,
+      "completions/max_length": 396.0,
+      "completions/max_terminated_length": 396.0,
+      "completions/mean_length": 88.14583587646484,
+      "completions/mean_terminated_length": 88.14583587646484,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.3161276876926422,
+      "epoch": 6.073170731707317,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07784765958786011,
+      "kl": 4.819030800717883e-05,
+      "learning_rate": 4.0480444541766575e-07,
+      "loss": 0.12299837917089462,
+      "num_tokens": 6151955.0,
+      "reward": 0.5446512699127197,
+      "reward_std": 0.2611033618450165,
+      "rewards/true_env_reward_fn/mean": 0.544651210308075,
+      "rewards/true_env_reward_fn/std": 0.2611033618450165,
       "step": 249,
-      "step_time": 8.798317029002646
+      "step_time": 25.232192139999825
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -6988,26 +6988,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 64.75,
-      "completions/mean_terminated_length": 64.75,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.37166029214859,
-      "epoch": 2.032520325203252,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1542455554008484,
-      "kl": 2.0379054603836266e-05,
-      "learning_rate": 2.313339466199264e-07,
-      "loss": -0.037539318203926086,
-      "num_tokens": 1025971.0,
-      "reward": 0.6065863966941833,
-      "reward_std": 0.032470256090164185,
-      "rewards/true_env_reward_fn/mean": 0.6065863966941833,
-      "rewards/true_env_reward_fn/std": 0.032470256090164185,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 69.97917175292969,
+      "completions/mean_terminated_length": 69.97917175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2318329215049744,
+      "epoch": 6.097560975609756,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06498592346906662,
+      "kl": 3.854301621686318e-05,
+      "learning_rate": 4.0062891124787e-07,
+      "loss": -0.04302535578608513,
+      "num_tokens": 6174898.0,
+      "reward": 0.6081289052963257,
+      "reward_std": 0.24437586963176727,
+      "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+      "rewards/true_env_reward_fn/std": 0.24437588453292847,
       "step": 250,
-      "step_time": 4.096263454999644
+      "step_time": 10.385816780999448
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7016,26 +7016,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 60.5,
-      "completions/mean_terminated_length": 60.5,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.4042693972587585,
-      "epoch": 2.040650406504065,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.163822203874588,
-      "kl": 3.525477495713858e-05,
-      "learning_rate": 2.2790944756396916e-07,
-      "loss": 0.03408379852771759,
-      "num_tokens": 1029415.0,
-      "reward": 0.37829869985580444,
-      "reward_std": 0.2773255407810211,
-      "rewards/true_env_reward_fn/mean": 0.37829869985580444,
-      "rewards/true_env_reward_fn/std": 0.2773255407810211,
+      "completions/max_length": 93.0,
+      "completions/max_terminated_length": 93.0,
+      "completions/mean_length": 60.91666793823242,
+      "completions/mean_terminated_length": 60.91666793823242,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2425517737865448,
+      "epoch": 6.121951219512195,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08402577042579651,
+      "kl": 0.00010545800250838511,
+      "learning_rate": 3.9646057992918966e-07,
+      "loss": 0.006328321993350983,
+      "num_tokens": 6203582.0,
+      "reward": 0.30250340700149536,
+      "reward_std": 0.3084425926208496,
+      "rewards/true_env_reward_fn/mean": 0.30250340700149536,
+      "rewards/true_env_reward_fn/std": 0.3084425628185272,
       "step": 251,
-      "step_time": 3.9464334140011488
+      "step_time": 9.779451584999606
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7044,26 +7044,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 181.0,
-      "completions/max_terminated_length": 181.0,
-      "completions/mean_length": 73.75,
-      "completions/mean_terminated_length": 73.75,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2268111109733582,
-      "epoch": 2.048780487804878,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1961166262626648,
-      "kl": 3.528672823449597e-05,
-      "learning_rate": 2.2450035761129391e-07,
-      "loss": 0.2999379336833954,
-      "num_tokens": 1032213.0,
-      "reward": 0.6772161722183228,
-      "reward_std": 0.31218820810317993,
-      "rewards/true_env_reward_fn/mean": 0.6772161722183228,
-      "rewards/true_env_reward_fn/std": 0.31218820810317993,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 65.5,
+      "completions/mean_terminated_length": 65.5,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2977190017700195,
+      "epoch": 6.146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08372989296913147,
+      "kl": 0.00017708011364447884,
+      "learning_rate": 3.9229975360050934e-07,
+      "loss": 0.04882372170686722,
+      "num_tokens": 6227486.0,
+      "reward": 0.5265982151031494,
+      "reward_std": 0.31736499071121216,
+      "rewards/true_env_reward_fn/mean": 0.5265981554985046,
+      "rewards/true_env_reward_fn/std": 0.31736496090888977,
       "step": 252,
-      "step_time": 7.437029113001699
+      "step_time": 10.318136508000407
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7072,26 +7072,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 59.625,
-      "completions/mean_terminated_length": 59.625,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.360656499862671,
-      "epoch": 2.0569105691056913,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24334783852100372,
-      "kl": 5.1042834456893615e-05,
-      "learning_rate": 2.2110698201409787e-07,
-      "loss": 0.05509951710700989,
-      "num_tokens": 1038598.0,
-      "reward": 0.2947666645050049,
-      "reward_std": 0.19891902804374695,
-      "rewards/true_env_reward_fn/mean": 0.2947666645050049,
-      "rewards/true_env_reward_fn/std": 0.19891902804374695,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 71.125,
+      "completions/mean_terminated_length": 71.125,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.3135383129119873,
+      "epoch": 6.170731707317073,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05451377481222153,
+      "kl": 6.562464477610774e-05,
+      "learning_rate": 3.8814673385671893e-07,
+      "loss": 0.01392640545964241,
+      "num_tokens": 6255436.0,
+      "reward": 0.4130678176879883,
+      "reward_std": 0.1573377549648285,
+      "rewards/true_env_reward_fn/mean": 0.4130678176879883,
+      "rewards/true_env_reward_fn/std": 0.15733776986598969,
       "step": 253,
-      "step_time": 3.7359043900014512
+      "step_time": 9.396596211999622
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7100,26 +7100,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 74.75,
-      "completions/mean_terminated_length": 74.75,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.4314632415771484,
-      "epoch": 2.065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1960419863462448,
-      "kl": 6.17889963905327e-05,
-      "learning_rate": 2.1772962461750342e-07,
-      "loss": 0.0954262986779213,
-      "num_tokens": 1043268.0,
-      "reward": 0.4036714732646942,
-      "reward_std": 0.42137831449508667,
-      "rewards/true_env_reward_fn/mean": 0.4036714732646942,
-      "rewards/true_env_reward_fn/std": 0.42137834429740906,
+      "completions/max_length": 240.0,
+      "completions/max_terminated_length": 240.0,
+      "completions/mean_length": 73.85417175292969,
+      "completions/mean_terminated_length": 73.85417175292969,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.3749704957008362,
+      "epoch": 6.195121951219512,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05664386227726936,
+      "kl": 6.744195252395002e-05,
+      "learning_rate": 3.840018217268526e-07,
+      "loss": -0.04205852374434471,
+      "num_tokens": 6281557.0,
+      "reward": 0.5188004970550537,
+      "reward_std": 0.2932124733924866,
+      "rewards/true_env_reward_fn/mean": 0.5188004970550537,
+      "rewards/true_env_reward_fn/std": 0.2932124435901642,
       "step": 254,
-      "step_time": 5.565175547999388
+      "step_time": 19.77463799499992
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7128,26 +7128,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 68.625,
-      "completions/mean_terminated_length": 68.625,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0521443486213684,
-      "epoch": 2.073170731707317,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.464648271910846e-05,
-      "kl": 1.2018902907584561e-05,
-      "learning_rate": 2.1436858783235338e-07,
-      "loss": 6.008343689245521e-07,
-      "num_tokens": 1046517.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/max_length": 140.0,
+      "completions/max_terminated_length": 140.0,
+      "completions/mean_length": 69.60417175292969,
+      "completions/mean_terminated_length": 69.60417175292969,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2866049408912659,
+      "epoch": 6.219512195121951,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06639927625656128,
+      "kl": 5.235667686065426e-05,
+      "learning_rate": 3.798653176522696e-07,
+      "loss": -0.019558893516659737,
+      "num_tokens": 6302850.0,
+      "reward": 0.5025100111961365,
+      "reward_std": 0.1717289537191391,
+      "rewards/true_env_reward_fn/mean": 0.5025100111961365,
+      "rewards/true_env_reward_fn/std": 0.1717289537191391,
       "step": 255,
-      "step_time": 4.379171047001364
+      "step_time": 9.634558264001043
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7156,26 +7156,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 55.25,
-      "completions/mean_terminated_length": 55.25,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.091518223285675,
-      "epoch": 2.08130081300813,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.263061656383798e-05,
-      "kl": 1.4573892713087844e-05,
-      "learning_rate": 2.110241726081317e-07,
-      "loss": 7.304333848878741e-07,
-      "num_tokens": 1050439.0,
-      "reward": 0.6024306416511536,
-      "reward_std": 0.13815335929393768,
-      "rewards/true_env_reward_fn/mean": 0.6024306416511536,
-      "rewards/true_env_reward_fn/std": 0.13815335929393768,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 66.0625,
+      "completions/mean_terminated_length": 66.0625,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.3105897009372711,
+      "epoch": 6.2439024390243905,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07659970223903656,
+      "kl": 8.12946273072157e-05,
+      "learning_rate": 3.7573752146487636e-07,
+      "loss": 0.05201243981719017,
+      "num_tokens": 6329269.0,
+      "reward": 0.5027985572814941,
+      "reward_std": 0.2521378695964813,
+      "rewards/true_env_reward_fn/mean": 0.5027985572814941,
+      "rewards/true_env_reward_fn/std": 0.2521378993988037,
       "step": 256,
-      "step_time": 3.8379976090000127
+      "step_time": 11.07390475200009
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7184,26 +7184,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 60.375,
-      "completions/mean_terminated_length": 60.375,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2301559448242188,
-      "epoch": 2.089430894308943,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.17180238664150238,
-      "kl": 2.236898035334889e-05,
-      "learning_rate": 2.076966784060165e-07,
-      "loss": 0.023751959204673767,
-      "num_tokens": 1054578.0,
-      "reward": 0.4112047851085663,
-      "reward_std": 0.05330020561814308,
-      "rewards/true_env_reward_fn/mean": 0.4112047851085663,
-      "rewards/true_env_reward_fn/std": 0.05330020561814308,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 56.25,
+      "completions/mean_terminated_length": 56.25,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 1.3154918253421783,
+      "epoch": 6.2682926829268295,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10612925887107849,
+      "kl": 0.00010100230429088697,
+      "learning_rate": 3.7161873236539386e-07,
+      "loss": 0.0214182510972023,
+      "num_tokens": 6356233.0,
+      "reward": 0.2817384600639343,
+      "reward_std": 0.3363598883152008,
+      "rewards/true_env_reward_fn/mean": 0.2817384600639343,
+      "rewards/true_env_reward_fn/std": 0.3363598883152008,
       "step": 257,
-      "step_time": 4.424114469000415
+      "step_time": 10.784447634000117
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7212,26 +7212,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 52.125,
-      "completions/mean_terminated_length": 52.125,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2734522223472595,
-      "epoch": 2.097560975609756,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2274128645658493,
-      "kl": 5.3426387239596806e-05,
-      "learning_rate": 2.043864031720667e-07,
-      "loss": 0.05165906995534897,
-      "num_tokens": 1060579.0,
-      "reward": 0.2129499912261963,
-      "reward_std": 0.2476053088903427,
-      "rewards/true_env_reward_fn/mean": 0.2129499912261963,
-      "rewards/true_env_reward_fn/std": 0.2476053088903427,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 62.770835876464844,
+      "completions/mean_terminated_length": 62.770835876464844,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.1986172497272491,
+      "epoch": 6.2926829268292686,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08129199594259262,
+      "kl": 0.00011449725025158841,
+      "learning_rate": 3.6750924890166923e-07,
+      "loss": -0.05941678211092949,
+      "num_tokens": 6375046.0,
+      "reward": 0.5760313272476196,
+      "reward_std": 0.24430812895298004,
+      "rewards/true_env_reward_fn/mean": 0.5760312676429749,
+      "rewards/true_env_reward_fn/std": 0.24430814385414124,
       "step": 258,
-      "step_time": 3.4830677139998443
+      "step_time": 9.467202022999572
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7240,26 +7240,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 54.375,
-      "completions/mean_terminated_length": 54.375,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2013150453567505,
-      "epoch": 2.105691056910569,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20796725153923035,
-      "kl": 8.34841157484334e-05,
-      "learning_rate": 2.0109364331054297e-07,
-      "loss": 0.04251064360141754,
-      "num_tokens": 1065318.0,
-      "reward": 0.2038009911775589,
-      "reward_std": 0.3393669128417969,
-      "rewards/true_env_reward_fn/mean": 0.2038009911775589,
-      "rewards/true_env_reward_fn/std": 0.3393669128417969,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 62.75,
+      "completions/mean_terminated_length": 62.75,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2520000636577606,
+      "epoch": 6.317073170731708,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.06876237690448761,
+      "kl": 0.0001509120993432589,
+      "learning_rate": 3.6340936894703713e-07,
+      "loss": 0.015932239592075348,
+      "num_tokens": 6397298.0,
+      "reward": 0.4944794774055481,
+      "reward_std": 0.24476772546768188,
+      "rewards/true_env_reward_fn/mean": 0.4944794476032257,
+      "rewards/true_env_reward_fn/std": 0.24476774036884308,
       "step": 259,
-      "step_time": 3.34712773299907
+      "step_time": 10.47640546699995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7268,26 +7268,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 66.0,
-      "completions/max_terminated_length": 66.0,
-      "completions/mean_length": 52.25,
-      "completions/mean_terminated_length": 52.25,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.3005307912826538,
-      "epoch": 2.113821138211382,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2342396229505539,
-      "kl": 2.9608699151140172e-05,
-      "learning_rate": 1.9781869365736777e-07,
-      "loss": -0.11542908847332001,
-      "num_tokens": 1068352.0,
-      "reward": 0.5755212306976318,
-      "reward_std": 0.1629202961921692,
-      "rewards/true_env_reward_fn/mean": 0.5755212306976318,
-      "rewards/true_env_reward_fn/std": 0.1629202961921692,
+      "completions/max_length": 125.0,
+      "completions/max_terminated_length": 125.0,
+      "completions/mean_length": 66.9375,
+      "completions/mean_terminated_length": 66.9375,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3675504922866821,
+      "epoch": 6.341463414634147,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06423573940992355,
+      "kl": 0.00010545238592385431,
+      "learning_rate": 3.593193896787277e-07,
+      "loss": 0.006066613830626011,
+      "num_tokens": 6423927.0,
+      "reward": 0.4633293151855469,
+      "reward_std": 0.2953638732433319,
+      "rewards/true_env_reward_fn/mean": 0.4633293151855469,
+      "rewards/true_env_reward_fn/std": 0.2953638732433319,
       "step": 260,
-      "step_time": 3.0931306170004973
+      "step_time": 11.672983966999709
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7296,26 +7296,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 70.875,
-      "completions/mean_terminated_length": 70.875,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1381222009658813,
-      "epoch": 2.1219512195121952,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13169968128204346,
-      "kl": 1.4705466128361877e-05,
-      "learning_rate": 1.9456184745372558e-07,
-      "loss": -0.026440951973199844,
-      "num_tokens": 1070847.0,
-      "reward": 0.5704532265663147,
-      "reward_std": 0.13928835093975067,
-      "rewards/true_env_reward_fn/mean": 0.5704532265663147,
-      "rewards/true_env_reward_fn/std": 0.13928835093975067,
+      "completions/max_length": 173.0,
+      "completions/max_terminated_length": 173.0,
+      "completions/mean_length": 74.5,
+      "completions/mean_terminated_length": 74.5,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3533118963241577,
+      "epoch": 6.365853658536586,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07830996066331863,
+      "kl": 8.717951732251095e-05,
+      "learning_rate": 3.552396075563257e-07,
+      "loss": -0.021772567182779312,
+      "num_tokens": 6451487.0,
+      "reward": 0.3543766736984253,
+      "reward_std": 0.33776554465293884,
+      "rewards/true_env_reward_fn/mean": 0.3543766736984253,
+      "rewards/true_env_reward_fn/std": 0.33776557445526123,
       "step": 261,
-      "step_time": 3.8980969309996
+      "step_time": 13.734938852000141
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7324,26 +7324,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 57.25,
-      "completions/mean_terminated_length": 57.25,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.241390347480774,
-      "epoch": 2.130081300813008,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010450400441186503,
-      "kl": 1.6406540453317575e-05,
-      "learning_rate": 1.9132339631980622e-07,
-      "loss": 8.202600838558283e-07,
-      "num_tokens": 1076153.0,
-      "reward": 0.41493332386016846,
-      "reward_std": 0.06228968873620033,
-      "rewards/true_env_reward_fn/mean": 0.41493332386016846,
-      "rewards/true_env_reward_fn/std": 0.06228969246149063,
+      "completions/max_length": 103.0,
+      "completions/max_terminated_length": 103.0,
+      "completions/mean_length": 71.45833587646484,
+      "completions/mean_terminated_length": 71.45833587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.1788119971752167,
+      "epoch": 6.390243902439025,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0994381383061409,
+      "kl": 0.00011037426884286106,
+      "learning_rate": 3.511703183002827e-07,
+      "loss": 0.008012240752577782,
+      "num_tokens": 6476141.0,
+      "reward": 0.45457857847213745,
+      "reward_std": 0.24262367188930511,
+      "rewards/true_env_reward_fn/mean": 0.45457854866981506,
+      "rewards/true_env_reward_fn/std": 0.24262367188930511,
       "step": 262,
-      "step_time": 3.6601423579995753
+      "step_time": 10.59573544200066
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7352,26 +7352,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 81.0,
-      "completions/mean_terminated_length": 81.0,
-      "completions/min_length": 67.0,
-      "completions/min_terminated_length": 67.0,
-      "entropy": 1.1326860189437866,
-      "epoch": 2.138211382113821,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001754833065206185,
-      "kl": 1.4390577689482598e-05,
-      "learning_rate": 1.881036302286923e-07,
-      "loss": 7.2446778176527e-07,
-      "num_tokens": 1080537.0,
-      "reward": 0.5930472612380981,
-      "reward_std": 0.14818456768989563,
-      "rewards/true_env_reward_fn/mean": 0.5930472612380981,
-      "rewards/true_env_reward_fn/std": 0.14818456768989563,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 58.1875,
+      "completions/mean_terminated_length": 58.1875,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.293170690536499,
+      "epoch": 6.414634146341464,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10550480335950851,
+      "kl": 0.00020241059792169835,
+      "learning_rate": 3.4711181687048106e-07,
+      "loss": 0.02680305391550064,
+      "num_tokens": 6499670.0,
+      "reward": 0.33261698484420776,
+      "reward_std": 0.31543657183647156,
+      "rewards/true_env_reward_fn/mean": 0.3326169550418854,
+      "rewards/true_env_reward_fn/std": 0.31543657183647156,
       "step": 263,
-      "step_time": 4.190891189999093
+      "step_time": 10.02452396199942
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7380,26 +7380,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 103.0,
-      "completions/max_terminated_length": 103.0,
-      "completions/mean_length": 58.875,
-      "completions/mean_terminated_length": 58.875,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.353486955165863,
-      "epoch": 2.1463414634146343,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.215454563498497,
-      "kl": 6.859865607111715e-05,
-      "learning_rate": 1.8490283748039515e-07,
-      "loss": 0.0181141197681427,
-      "num_tokens": 1084056.0,
-      "reward": 0.3049938380718231,
-      "reward_std": 0.4605039656162262,
-      "rewards/true_env_reward_fn/mean": 0.3049938380718231,
-      "rewards/true_env_reward_fn/std": 0.4605039954185486,
+      "completions/max_length": 348.0,
+      "completions/max_terminated_length": 348.0,
+      "completions/mean_length": 89.83333587646484,
+      "completions/mean_terminated_length": 89.83333587646484,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2982321977615356,
+      "epoch": 6.439024390243903,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06412192434072495,
+      "kl": 6.536830551340245e-05,
+      "learning_rate": 3.4306439744485447e-07,
+      "loss": -0.08396965265274048,
+      "num_tokens": 6522630.0,
+      "reward": 0.5518762469291687,
+      "reward_std": 0.21314994990825653,
+      "rewards/true_env_reward_fn/mean": 0.5518762469291687,
+      "rewards/true_env_reward_fn/std": 0.21314994990825653,
       "step": 264,
-      "step_time": 4.504906432999633
+      "step_time": 20.82168071200067
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7408,26 +7408,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 127.0,
-      "completions/max_terminated_length": 127.0,
-      "completions/mean_length": 70.375,
-      "completions/mean_terminated_length": 70.375,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 63.85416793823242,
+      "completions/mean_terminated_length": 63.85416793823242,
       "completions/min_length": 41.0,
       "completions/min_terminated_length": 41.0,
-      "entropy": 1.0113105773925781,
-      "epoch": 2.154471544715447,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10072485357522964,
-      "kl": 1.4604076568502933e-05,
-      "learning_rate": 1.8172130467604085e-07,
-      "loss": -0.041721273213624954,
-      "num_tokens": 1090171.0,
-      "reward": 0.47745320200920105,
-      "reward_std": 0.2929421067237854,
-      "rewards/true_env_reward_fn/mean": 0.47745320200920105,
-      "rewards/true_env_reward_fn/std": 0.2929421067237854,
+      "entropy": 1.2279660403728485,
+      "epoch": 6.463414634146342,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10119301080703735,
+      "kl": 0.00010847221983567579,
+      "learning_rate": 3.3902835339806456e-07,
+      "loss": 0.01589711755514145,
+      "num_tokens": 6548183.0,
+      "reward": 0.4527897238731384,
+      "reward_std": 0.2534019947052002,
+      "rewards/true_env_reward_fn/mean": 0.45278969407081604,
+      "rewards/true_env_reward_fn/std": 0.2534019649028778,
       "step": 265,
-      "step_time": 6.299696521999067
+      "step_time": 8.63894235699945
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7436,26 +7436,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 57.375,
-      "completions/mean_terminated_length": 57.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.3366597294807434,
-      "epoch": 2.16260162601626,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2297964245080948,
-      "kl": 5.677436638507061e-05,
-      "learning_rate": 1.785593166922062e-07,
-      "loss": 0.20361776649951935,
-      "num_tokens": 1094358.0,
-      "reward": 0.3835672438144684,
-      "reward_std": 0.4339357614517212,
-      "rewards/true_env_reward_fn/mean": 0.3835672438144684,
-      "rewards/true_env_reward_fn/std": 0.4339357912540436,
+      "completions/max_length": 99.0,
+      "completions/max_terminated_length": 99.0,
+      "completions/mean_length": 60.54166793823242,
+      "completions/mean_terminated_length": 60.54166793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.1612891554832458,
+      "epoch": 6.487804878048781,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0825633853673935,
+      "kl": 7.345602170971688e-05,
+      "learning_rate": 3.3500397728023534e-07,
+      "loss": 0.057398565113544464,
+      "num_tokens": 6571537.0,
+      "reward": 0.5196421146392822,
+      "reward_std": 0.18547315895557404,
+      "rewards/true_env_reward_fn/mean": 0.5196421146392822,
+      "rewards/true_env_reward_fn/std": 0.18547315895557404,
       "step": 266,
-      "step_time": 4.192992550000781
+      "step_time": 10.01155260700034
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7464,26 +7464,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 65.375,
-      "completions/mean_terminated_length": 65.375,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.4040917754173279,
-      "epoch": 2.1707317073170733,
+      "completions/max_length": 253.0,
+      "completions/max_terminated_length": 253.0,
+      "completions/mean_length": 67.85417175292969,
+      "completions/mean_terminated_length": 67.85417175292969,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2408597469329834,
+      "epoch": 6.512195121951219,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1913405954837799,
-      "kl": 2.2514723241329193e-05,
-      "learning_rate": 1.7541715665541276e-07,
-      "loss": -0.12278837710618973,
-      "num_tokens": 1099685.0,
-      "reward": 0.5316476821899414,
-      "reward_std": 0.1867343932390213,
-      "rewards/true_env_reward_fn/mean": 0.5316476821899414,
-      "rewards/true_env_reward_fn/std": 0.1867344230413437,
+      "grad_norm": 0.09217255562543869,
+      "kl": 9.46905902310391e-05,
+      "learning_rate": 3.3099156079574867e-07,
+      "loss": 0.026750221848487854,
+      "num_tokens": 6596010.0,
+      "reward": 0.4330406188964844,
+      "reward_std": 0.20423907041549683,
+      "rewards/true_env_reward_fn/mean": 0.4330406188964844,
+      "rewards/true_env_reward_fn/std": 0.20423908531665802,
       "step": 267,
-      "step_time": 4.577502725998784
+      "step_time": 20.002466699999786
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7492,26 +7492,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 100.0,
-      "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2707499265670776,
-      "epoch": 2.178861788617886,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1252160668373108,
-      "kl": 2.1676510186807718e-05,
-      "learning_rate": 1.722951059167737e-07,
-      "loss": 0.030697450041770935,
-      "num_tokens": 1103244.0,
-      "reward": 0.47949954867362976,
-      "reward_std": 0.10323704034090042,
-      "rewards/true_env_reward_fn/mean": 0.47949954867362976,
-      "rewards/true_env_reward_fn/std": 0.10323705524206161,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 64.83333587646484,
+      "completions/mean_terminated_length": 64.83333587646484,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2907497882843018,
+      "epoch": 6.536585365853659,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07903403043746948,
+      "kl": 0.00015101409735507332,
+      "learning_rate": 3.269913947820998e-07,
+      "loss": 0.02006390690803528,
+      "num_tokens": 6622106.0,
+      "reward": 0.4544076919555664,
+      "reward_std": 0.26717478036880493,
+      "rewards/true_env_reward_fn/mean": 0.4544076919555664,
+      "rewards/true_env_reward_fn/std": 0.2671748101711273,
       "step": 268,
-      "step_time": 4.3898782989999745
+      "step_time": 8.818348709999555
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7520,26 +7520,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.3778526186943054,
-      "epoch": 2.186991869918699,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19924476742744446,
-      "kl": 5.189802323002368e-05,
-      "learning_rate": 1.6919344402680231e-07,
-      "loss": 0.03015017881989479,
-      "num_tokens": 1108318.0,
-      "reward": 0.12559716403484344,
-      "reward_std": 0.22310735285282135,
-      "rewards/true_env_reward_fn/mean": 0.12559716403484344,
-      "rewards/true_env_reward_fn/std": 0.22310735285282135,
+      "completions/max_length": 138.0,
+      "completions/max_terminated_length": 138.0,
+      "completions/mean_length": 62.0625,
+      "completions/mean_terminated_length": 62.0625,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2150432765483856,
+      "epoch": 6.560975609756097,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09289928525686264,
+      "kl": 9.474463149672374e-05,
+      "learning_rate": 3.230037691888162e-07,
+      "loss": 0.08772514015436172,
+      "num_tokens": 6636101.0,
+      "reward": 0.6059033870697021,
+      "reward_std": 0.23812197148799896,
+      "rewards/true_env_reward_fn/mean": 0.6059033274650574,
+      "rewards/true_env_reward_fn/std": 0.23812197148799896,
       "step": 269,
-      "step_time": 4.379851057999986
+      "step_time": 8.47666211500109
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7548,26 +7548,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.0,
-      "completions/max_terminated_length": 64.0,
-      "completions/mean_length": 58.875,
-      "completions/mean_terminated_length": 58.875,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.2916911840438843,
-      "epoch": 2.1951219512195124,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18381677567958832,
-      "kl": 2.593698127384414e-05,
-      "learning_rate": 1.6611244871038116e-07,
-      "loss": 0.025029506534337997,
-      "num_tokens": 1116025.0,
-      "reward": 0.05200198292732239,
-      "reward_std": 0.276480108499527,
-      "rewards/true_env_reward_fn/mean": 0.05200198292732239,
-      "rewards/true_env_reward_fn/std": 0.276480108499527,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 62.5,
+      "completions/mean_terminated_length": 62.5,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "entropy": 1.2745259404182434,
+      "epoch": 6.585365853658536,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09368392825126648,
+      "kl": 0.0001518711451353738,
+      "learning_rate": 3.1902897305644093e-07,
+      "loss": -0.003839995712041855,
+      "num_tokens": 6659877.0,
+      "reward": 0.5029901266098022,
+      "reward_std": 0.3467065095901489,
+      "rewards/true_env_reward_fn/mean": 0.5029900670051575,
+      "rewards/true_env_reward_fn/std": 0.3467065393924713,
       "step": 270,
-      "step_time": 3.6788300769985653
+      "step_time": 11.298448464000103
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7576,26 +7576,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 215.0,
-      "completions/max_terminated_length": 215.0,
-      "completions/mean_length": 94.625,
-      "completions/mean_terminated_length": 94.625,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.2625537514686584,
-      "epoch": 2.203252032520325,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.09628148376941681,
-      "kl": 2.636932003952097e-05,
-      "learning_rate": 1.6305239584189344e-07,
-      "loss": 0.009904414415359497,
-      "num_tokens": 1120434.0,
-      "reward": 0.5924437046051025,
-      "reward_std": 0.39917245507240295,
-      "rewards/true_env_reward_fn/mean": 0.5924437046051025,
-      "rewards/true_env_reward_fn/std": 0.39917245507240295,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 67.0625,
+      "completions/mean_terminated_length": 67.0625,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 1.238816499710083,
+      "epoch": 6.609756097560975,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.05819084122776985,
+      "kl": 6.655609695371822e-05,
+      "learning_rate": 3.150672944955818e-07,
+      "loss": -0.02771003544330597,
+      "num_tokens": 6679776.0,
+      "reward": 0.5772091746330261,
+      "reward_std": 0.17815756797790527,
+      "rewards/true_env_reward_fn/mean": 0.5772091746330261,
+      "rewards/true_env_reward_fn/std": 0.17815756797790527,
       "step": 271,
-      "step_time": 9.09279120499923
+      "step_time": 7.902968623000561
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7604,26 +7604,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 58.875,
-      "completions/mean_terminated_length": 58.875,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.2491654753684998,
-      "epoch": 2.2113821138211383,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00015083610196597874,
-      "kl": 1.4522283436235739e-05,
-      "learning_rate": 1.6001355942052182e-07,
-      "loss": 7.247089683914965e-07,
-      "num_tokens": 1122349.0,
-      "reward": 0.812765896320343,
-      "reward_std": 0.04731824994087219,
-      "rewards/true_env_reward_fn/mean": 0.812765896320343,
-      "rewards/true_env_reward_fn/std": 0.04731824994087219,
+      "completions/max_length": 145.0,
+      "completions/max_terminated_length": 145.0,
+      "completions/mean_length": 75.10417175292969,
+      "completions/mean_terminated_length": 75.10417175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3645851016044617,
+      "epoch": 6.634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07474905252456665,
+      "kl": 7.497054593841312e-05,
+      "learning_rate": 3.1111902066602724e-07,
+      "loss": 0.07271203398704529,
+      "num_tokens": 6704717.0,
+      "reward": 0.42504438757896423,
+      "reward_std": 0.2500284016132355,
+      "rewards/true_env_reward_fn/mean": 0.42504438757896423,
+      "rewards/true_env_reward_fn/std": 0.2500284016132355,
       "step": 272,
-      "step_time": 3.7730076539992297
+      "step_time": 11.259095110999624
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7632,26 +7632,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 71.25,
-      "completions/mean_terminated_length": 71.25,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.1462301015853882,
-      "epoch": 2.2195121951219514,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00012917000276502222,
-      "kl": 1.3649782886204775e-05,
-      "learning_rate": 1.569962115457138e-07,
-      "loss": 6.695274805679219e-07,
-      "num_tokens": 1124831.0,
-      "reward": 0.7329437732696533,
-      "reward_std": 0.22123214602470398,
-      "rewards/true_env_reward_fn/mean": 0.7329437732696533,
-      "rewards/true_env_reward_fn/std": 0.22123214602470398,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 67.64583587646484,
+      "completions/mean_terminated_length": 67.64583587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2410458326339722,
+      "epoch": 6.658536585365853,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0750170573592186,
+      "kl": 8.587932643422391e-05,
+      "learning_rate": 3.0718443775593225e-07,
+      "loss": -0.019169267266988754,
+      "num_tokens": 6727060.0,
+      "reward": 0.5114523768424988,
+      "reward_std": 0.17165428400039673,
+      "rewards/true_env_reward_fn/mean": 0.5114523768424988,
+      "rewards/true_env_reward_fn/std": 0.17165428400039673,
       "step": 273,
-      "step_time": 3.907510233000721
+      "step_time": 11.097374408000178
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7660,26 +7660,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 175.0,
-      "completions/max_terminated_length": 175.0,
-      "completions/mean_length": 59.25,
-      "completions/mean_terminated_length": 59.25,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2922418713569641,
-      "epoch": 2.227642276422764,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15563185513019562,
-      "kl": 3.604595076467376e-05,
-      "learning_rate": 1.5400062239281858e-07,
-      "loss": -0.034219659864902496,
-      "num_tokens": 1128837.0,
-      "reward": 0.46255773305892944,
-      "reward_std": 0.24893923103809357,
-      "rewards/true_env_reward_fn/mean": 0.46255773305892944,
-      "rewards/true_env_reward_fn/std": 0.24893923103809357,
+      "completions/max_length": 130.0,
+      "completions/max_terminated_length": 130.0,
+      "completions/mean_length": 67.08333587646484,
+      "completions/mean_terminated_length": 67.08333587646484,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "entropy": 1.2417791783809662,
+      "epoch": 6.682926829268292,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08969064801931381,
+      "kl": 5.61167857995315e-05,
+      "learning_rate": 3.0326383096107423e-07,
+      "loss": 0.11341358721256256,
+      "num_tokens": 6752824.0,
+      "reward": 0.4772833287715912,
+      "reward_std": 0.31618404388427734,
+      "rewards/true_env_reward_fn/mean": 0.4772833287715912,
+      "rewards/true_env_reward_fn/std": 0.31618407368659973,
       "step": 274,
-      "step_time": 7.502110859999448
+      "step_time": 11.189890726001067
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7688,26 +7688,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 66.0,
-      "completions/max_terminated_length": 66.0,
-      "completions/mean_length": 50.75,
-      "completions/mean_terminated_length": 50.75,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.2709790468215942,
-      "epoch": 2.2357723577235773,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 67.47917175292969,
+      "completions/mean_terminated_length": 67.47917175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.272821843624115,
+      "epoch": 6.7073170731707314,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24958185851573944,
-      "kl": 6.180045966175385e-05,
-      "learning_rate": 1.5102706018889428e-07,
-      "loss": -0.10148808360099792,
-      "num_tokens": 1132891.0,
-      "reward": 0.4240284562110901,
-      "reward_std": 0.43752968311309814,
-      "rewards/true_env_reward_fn/mean": 0.4240284562110901,
-      "rewards/true_env_reward_fn/std": 0.43752965331077576,
+      "grad_norm": 0.08874963968992233,
+      "kl": 0.00015055539006425533,
+      "learning_rate": 2.9935748446418065e-07,
+      "loss": 0.02028803899884224,
+      "num_tokens": 6776703.0,
+      "reward": 0.49093031883239746,
+      "reward_std": 0.25163188576698303,
+      "rewards/true_env_reward_fn/mean": 0.49093031883239746,
+      "rewards/true_env_reward_fn/std": 0.25163188576698303,
       "step": 275,
-      "step_time": 3.251475233999372
+      "step_time": 9.941926390000845
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7716,26 +7716,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 55.5,
-      "completions/mean_terminated_length": 55.5,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.2637454867362976,
-      "epoch": 2.2439024390243905,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.439944602083415e-05,
-      "kl": 1.127877567341784e-05,
-      "learning_rate": 1.4807579118869146e-07,
-      "loss": 5.64579522688291e-07,
-      "num_tokens": 1137611.0,
-      "reward": 0.4544333219528198,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.4544333219528198,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 365.0,
+      "completions/max_terminated_length": 365.0,
+      "completions/mean_length": 68.04167175292969,
+      "completions/mean_terminated_length": 68.04167175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.0912510752677917,
+      "epoch": 6.7317073170731705,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.060312703251838684,
+      "kl": 6.840269179519964e-05,
+      "learning_rate": 2.9546568141433e-07,
+      "loss": -0.036469291895627975,
+      "num_tokens": 6799417.0,
+      "reward": 0.5543485879898071,
+      "reward_std": 0.22147472202777863,
+      "rewards/true_env_reward_fn/mean": 0.5543485283851624,
+      "rewards/true_env_reward_fn/std": 0.22147469222545624,
       "step": 276,
-      "step_time": 3.311975311999049
+      "step_time": 22.291117544999906
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7744,26 +7744,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.1065265536308289,
-      "epoch": 2.252032520325203,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13068579137325287,
-      "kl": 4.4293181417742744e-05,
-      "learning_rate": 1.4514707965081262e-07,
-      "loss": 0.04848391190171242,
-      "num_tokens": 1142087.0,
-      "reward": 0.2852628827095032,
-      "reward_std": 0.23009054362773895,
-      "rewards/true_env_reward_fn/mean": 0.2852628827095032,
-      "rewards/true_env_reward_fn/std": 0.23009057343006134,
+      "completions/max_length": 90.0,
+      "completions/max_terminated_length": 90.0,
+      "completions/mean_length": 59.895835876464844,
+      "completions/mean_terminated_length": 59.895835876464844,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 1.2566059231758118,
+      "epoch": 6.7560975609756095,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09982399642467499,
+      "kl": 0.00020091429905733094,
+      "learning_rate": 2.9158870390642863e-07,
+      "loss": -0.005738064646720886,
+      "num_tokens": 6829804.0,
+      "reward": 0.24754562973976135,
+      "reward_std": 0.23038579523563385,
+      "rewards/true_env_reward_fn/mean": 0.24754561483860016,
+      "rewards/true_env_reward_fn/std": 0.23038578033447266,
       "step": 277,
-      "step_time": 3.9521008399988204
+      "step_time": 9.732460060999983
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7772,26 +7772,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 168.0,
-      "completions/max_terminated_length": 168.0,
-      "completions/mean_length": 104.5,
-      "completions/mean_terminated_length": 104.5,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.5374161005020142,
-      "epoch": 2.2601626016260163,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.129844531416893,
-      "kl": 3.9368313082377426e-05,
-      "learning_rate": 1.4224118781404917e-07,
-      "loss": -0.1482687145471573,
-      "num_tokens": 1147731.0,
-      "reward": 0.43388551473617554,
-      "reward_std": 0.39525240659713745,
-      "rewards/true_env_reward_fn/mean": 0.43388551473617554,
-      "rewards/true_env_reward_fn/std": 0.39525243639945984,
+      "completions/max_length": 149.0,
+      "completions/max_terminated_length": 149.0,
+      "completions/mean_length": 71.60417175292969,
+      "completions/mean_terminated_length": 71.60417175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2196767628192902,
+      "epoch": 6.780487804878049,
+      "frac_reward_zero_std": 0.8333333730697632,
+      "grad_norm": 0.03178449347615242,
+      "kl": 5.627466816804372e-05,
+      "learning_rate": 2.8772683296076194e-07,
+      "loss": 0.002586783841252327,
+      "num_tokens": 6852617.0,
+      "reward": 0.5281299352645874,
+      "reward_std": 0.22268518805503845,
+      "rewards/true_env_reward_fn/mean": 0.5281298756599426,
+      "rewards/true_env_reward_fn/std": 0.22268518805503845,
       "step": 278,
-      "step_time": 7.55689369099855
+      "step_time": 11.629210506000163
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7800,26 +7800,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 55.375,
-      "completions/mean_terminated_length": 55.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.0975646376609802,
-      "epoch": 2.2682926829268295,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1335010528564453,
-      "kl": 2.2682882445224095e-05,
-      "learning_rate": 1.3935837587390214e-07,
-      "loss": -0.009766265749931335,
-      "num_tokens": 1151246.0,
-      "reward": 0.37655720114707947,
-      "reward_std": 0.37258440256118774,
-      "rewards/true_env_reward_fn/mean": 0.37655720114707947,
-      "rewards/true_env_reward_fn/std": 0.37258440256118774,
+      "completions/max_length": 169.0,
+      "completions/max_terminated_length": 169.0,
+      "completions/mean_length": 68.66667175292969,
+      "completions/mean_terminated_length": 68.66667175292969,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.1991542279720306,
+      "epoch": 6.804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07567304372787476,
+      "kl": 0.00015181046910583973,
+      "learning_rate": 2.8388034850262646e-07,
+      "loss": -0.04906900227069855,
+      "num_tokens": 6881529.0,
+      "reward": 0.3906375765800476,
+      "reward_std": 0.3576822578907013,
+      "rewards/true_env_reward_fn/mean": 0.3906375467777252,
+      "rewards/true_env_reward_fn/std": 0.3576822876930237,
       "step": 279,
-      "step_time": 3.8792882219986495
+      "step_time": 18.253660386000774
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7828,26 +7828,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 63.5,
-      "completions/mean_terminated_length": 63.5,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.1846895217895508,
-      "epoch": 2.2764227642276422,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.138702854514122,
-      "kl": 2.0143882466072682e-05,
-      "learning_rate": 1.3649890195928254e-07,
-      "loss": 0.013748884201049805,
-      "num_tokens": 1153222.0,
-      "reward": 0.8218116760253906,
-      "reward_std": 0.1293545663356781,
-      "rewards/true_env_reward_fn/mean": 0.8218116760253906,
-      "rewards/true_env_reward_fn/std": 0.1293545812368393,
+      "completions/max_length": 208.0,
+      "completions/max_terminated_length": 208.0,
+      "completions/mean_length": 70.95833587646484,
+      "completions/mean_terminated_length": 70.95833587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.263102412223816,
+      "epoch": 6.829268292682927,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09104589372873306,
+      "kl": 0.00012209633769089123,
+      "learning_rate": 2.8004952934203837e-07,
+      "loss": 0.06642289459705353,
+      "num_tokens": 6905111.0,
+      "reward": 0.3988339304924011,
+      "reward_std": 0.34396088123321533,
+      "rewards/true_env_reward_fn/mean": 0.39883390069007874,
+      "rewards/true_env_reward_fn/std": 0.3439609110355377,
       "step": 280,
-      "step_time": 3.42550413599929
+      "step_time": 15.519001798998943
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7856,26 +7856,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 64.0,
-      "completions/mean_terminated_length": 64.0,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.3451175689697266,
-      "epoch": 2.2845528455284554,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1508527249097824,
-      "kl": 1.6968931049632374e-05,
-      "learning_rate": 1.336630221093991e-07,
-      "loss": 0.04001428931951523,
-      "num_tokens": 1157606.0,
-      "reward": 0.4994586706161499,
-      "reward_std": 0.10593737661838531,
-      "rewards/true_env_reward_fn/mean": 0.4994586706161499,
-      "rewards/true_env_reward_fn/std": 0.10593737661838531,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 59.0625,
+      "completions/mean_terminated_length": 59.0625,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.208267629146576,
+      "epoch": 6.853658536585366,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07465670257806778,
+      "kl": 7.14320785846212e-05,
+      "learning_rate": 2.762346531535246e-07,
+      "loss": 0.04124641790986061,
+      "num_tokens": 6926634.0,
+      "reward": 0.5870868563652039,
+      "reward_std": 0.28217118978500366,
+      "rewards/true_env_reward_fn/mean": 0.5870868563652039,
+      "rewards/true_env_reward_fn/std": 0.2821711599826813,
       "step": 281,
-      "step_time": 3.914840199000537
+      "step_time": 11.224198447001072
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7884,26 +7884,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 58.75,
-      "completions/mean_terminated_length": 58.75,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2436452507972717,
-      "epoch": 2.292682926829268,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12380471080541611,
-      "kl": 4.350653580331709e-05,
-      "learning_rate": 1.3085099025083245e-07,
-      "loss": -0.029160797595977783,
-      "num_tokens": 1162012.0,
-      "reward": 0.5066306591033936,
-      "reward_std": 0.28914663195610046,
-      "rewards/true_env_reward_fn/mean": 0.5066306591033936,
-      "rewards/true_env_reward_fn/std": 0.28914666175842285,
+      "completions/max_length": 162.0,
+      "completions/max_terminated_length": 162.0,
+      "completions/mean_length": 72.02083587646484,
+      "completions/mean_terminated_length": 72.02083587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.3142951428890228,
+      "epoch": 6.878048780487805,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07364725321531296,
+      "kl": 0.00011016946336894762,
+      "learning_rate": 2.7243599645599574e-07,
+      "loss": 0.004198473412543535,
+      "num_tokens": 6960219.0,
+      "reward": 0.34401482343673706,
+      "reward_std": 0.24699951708316803,
+      "rewards/true_env_reward_fn/mean": 0.3440147936344147,
+      "rewards/true_env_reward_fn/std": 0.24699951708316803,
       "step": 282,
-      "step_time": 3.9093819319987233
+      "step_time": 15.210776117999558
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7912,26 +7912,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 59.375,
-      "completions/mean_terminated_length": 59.375,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.10621577501297,
-      "epoch": 2.3008130081300813,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001239393459400162,
-      "kl": 1.6069413049990544e-05,
-      "learning_rate": 1.2806305817479771e-07,
-      "loss": 8.174432082341809e-07,
-      "num_tokens": 1165367.0,
-      "reward": 0.6557307243347168,
-      "reward_std": 0.2151959389448166,
-      "rewards/true_env_reward_fn/mean": 0.6557307243347168,
-      "rewards/true_env_reward_fn/std": 0.21519595384597778,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 60.583335876464844,
+      "completions/mean_terminated_length": 60.583335876464844,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2013934552669525,
+      "epoch": 6.902439024390244,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07040710747241974,
+      "kl": 7.988750212462037e-05,
+      "learning_rate": 2.686538345927026e-07,
+      "loss": 0.06334929168224335,
+      "num_tokens": 6982879.0,
+      "reward": 0.5139331221580505,
+      "reward_std": 0.19786590337753296,
+      "rewards/true_env_reward_fn/mean": 0.5139331221580505,
+      "rewards/true_env_reward_fn/std": 0.19786591827869415,
       "step": 283,
-      "step_time": 3.8117841049988783
+      "step_time": 10.240000448000501
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7940,26 +7940,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 66.0,
-      "completions/mean_terminated_length": 66.0,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.382466197013855,
-      "epoch": 2.3089430894308944,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1446879506111145,
-      "kl": 3.576014205464162e-05,
-      "learning_rate": 1.2529947551459964e-07,
-      "loss": -0.04731176793575287,
-      "num_tokens": 1169243.0,
-      "reward": 0.496622234582901,
-      "reward_std": 0.17860308289527893,
-      "rewards/true_env_reward_fn/mean": 0.496622234582901,
-      "rewards/true_env_reward_fn/std": 0.17860306799411774,
+      "completions/max_length": 103.0,
+      "completions/max_terminated_length": 103.0,
+      "completions/mean_length": 60.8125,
+      "completions/mean_terminated_length": 60.8125,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2223551571369171,
+      "epoch": 6.926829268292683,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07971613109111786,
+      "kl": 0.00016717875041649677,
+      "learning_rate": 2.64888441711279e-07,
+      "loss": 0.03706979751586914,
+      "num_tokens": 7003990.0,
+      "reward": 0.5301157832145691,
+      "reward_std": 0.2464885115623474,
+      "rewards/true_env_reward_fn/mean": 0.5301157832145691,
+      "rewards/true_env_reward_fn/std": 0.2464885115623474,
       "step": 284,
-      "step_time": 3.649135475998264
+      "step_time": 9.309556909001913
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7968,26 +7968,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 151.0,
-      "completions/max_terminated_length": 151.0,
-      "completions/mean_length": 61.375,
-      "completions/mean_terminated_length": 61.375,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.1946157813072205,
-      "epoch": 2.317073170731707,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10612626373767853,
-      "kl": 2.0257402866263874e-05,
-      "learning_rate": 1.2256048972327967e-07,
-      "loss": -0.029511645436286926,
-      "num_tokens": 1173594.0,
-      "reward": 0.5235810279846191,
-      "reward_std": 0.2810492217540741,
-      "rewards/true_env_reward_fn/mean": 0.5235810279846191,
-      "rewards/true_env_reward_fn/std": 0.2810492217540741,
+      "completions/max_length": 90.0,
+      "completions/max_terminated_length": 90.0,
+      "completions/mean_length": 57.97916793823242,
+      "completions/mean_terminated_length": 57.97916793823242,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.309690535068512,
+      "epoch": 6.951219512195122,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09541535377502441,
+      "kl": 0.00014337312313728034,
+      "learning_rate": 2.6114009074386844e-07,
+      "loss": -0.04744558781385422,
+      "num_tokens": 7027733.0,
+      "reward": 0.468932569026947,
+      "reward_std": 0.2396899163722992,
+      "rewards/true_env_reward_fn/mean": 0.46893253922462463,
+      "rewards/true_env_reward_fn/std": 0.2396899312734604,
       "step": 285,
-      "step_time": 6.7011265000001
+      "step_time": 8.91306197799986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -7996,26 +7996,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 68.75,
-      "completions/mean_terminated_length": 68.75,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2879216074943542,
-      "epoch": 2.3252032520325203,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12234477698802948,
-      "kl": 1.4488860415440286e-05,
-      "learning_rate": 1.1984634605145978e-07,
-      "loss": -0.05451745539903641,
-      "num_tokens": 1176384.0,
-      "reward": 0.6496104001998901,
-      "reward_std": 0.3368098735809326,
-      "rewards/true_env_reward_fn/mean": 0.6496104001998901,
-      "rewards/true_env_reward_fn/std": 0.3368098735809326,
+      "completions/max_length": 98.0,
+      "completions/max_terminated_length": 98.0,
+      "completions/mean_length": 64.75,
+      "completions/mean_terminated_length": 64.75,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.1709823310375214,
+      "epoch": 6.975609756097561,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.048187412321567535,
+      "kl": 9.718035107653122e-05,
+      "learning_rate": 2.5740905338734306e-07,
+      "loss": 0.010684527456760406,
+      "num_tokens": 7058137.0,
+      "reward": 0.44316989183425903,
+      "reward_std": 0.14355739951133728,
+      "rewards/true_env_reward_fn/mean": 0.44316986203193665,
+      "rewards/true_env_reward_fn/std": 0.14355739951133728,
       "step": 286,
-      "step_time": 4.18911992899848
+      "step_time": 9.546786461999545
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8024,26 +8024,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 105.0,
-      "completions/max_terminated_length": 105.0,
-      "completions/mean_length": 70.875,
-      "completions/mean_terminated_length": 70.875,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.0649144053459167,
-      "epoch": 2.3333333333333335,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1467445194721222,
-      "kl": 3.947542245441582e-05,
-      "learning_rate": 1.17157287525381e-07,
-      "loss": -0.029090911149978638,
-      "num_tokens": 1179651.0,
-      "reward": 0.4754716753959656,
-      "reward_std": 0.2500669062137604,
-      "rewards/true_env_reward_fn/mean": 0.4754716753959656,
-      "rewards/true_env_reward_fn/std": 0.2500669062137604,
+      "completions/max_length": 458.0,
+      "completions/max_terminated_length": 458.0,
+      "completions/mean_length": 77.97917175292969,
+      "completions/mean_terminated_length": 77.97917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3698437809944153,
+      "epoch": 7.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05576475337147713,
+      "kl": 9.964485252567101e-05,
+      "learning_rate": 2.536956000836082e-07,
+      "loss": 0.08634226769208908,
+      "num_tokens": 7082840.0,
+      "reward": 0.5030691027641296,
+      "reward_std": 0.2275215983390808,
+      "rewards/true_env_reward_fn/mean": 0.5030691027641296,
+      "rewards/true_env_reward_fn/std": 0.22752158343791962,
       "step": 287,
-      "step_time": 4.5810332049986755
+      "step_time": 26.703723129000537
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8052,26 +8052,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "entropy": 1.002779871225357,
-      "epoch": 2.341463414634146,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12533478438854218,
-      "kl": 2.3054610210238025e-05,
-      "learning_rate": 1.1449355492514437e-07,
-      "loss": -0.04500773549079895,
-      "num_tokens": 1181547.0,
-      "reward": 0.7334807515144348,
-      "reward_std": 0.12557923793792725,
-      "rewards/true_env_reward_fn/mean": 0.7334807515144348,
-      "rewards/true_env_reward_fn/std": 0.12557923793792725,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 58.1875,
+      "completions/mean_terminated_length": 58.1875,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2532718479633331,
+      "epoch": 7.024390243902439,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08579511940479279,
+      "kl": 9.319775290350663e-05,
+      "learning_rate": 2.500000000000001e-07,
+      "loss": -0.013113420456647873,
+      "num_tokens": 7098689.0,
+      "reward": 0.5958684682846069,
+      "reward_std": 0.1863170713186264,
+      "rewards/true_env_reward_fn/mean": 0.5958684682846069,
+      "rewards/true_env_reward_fn/std": 0.1863170564174652,
       "step": 288,
-      "step_time": 3.8661079009998502
+      "step_time": 7.618657231000725
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8080,26 +8080,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 294.0,
-      "completions/max_terminated_length": 294.0,
-      "completions/mean_length": 81.25,
-      "completions/mean_terminated_length": 81.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2280530333518982,
-      "epoch": 2.3495934959349594,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23682235181331635,
-      "kl": 5.290110129863024e-05,
-      "learning_rate": 1.1185538676315052e-07,
-      "loss": 0.5538168549537659,
-      "num_tokens": 1183973.0,
-      "reward": 0.5785378217697144,
-      "reward_std": 0.29215970635414124,
-      "rewards/true_env_reward_fn/mean": 0.5785378217697144,
-      "rewards/true_env_reward_fn/std": 0.2921597361564636,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 68.20833587646484,
+      "completions/mean_terminated_length": 68.20833587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.1685407161712646,
+      "epoch": 7.048780487804878,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07076086848974228,
+      "kl": 7.435419411194744e-05,
+      "learning_rate": 2.4632252100977564e-07,
+      "loss": 0.010196422226727009,
+      "num_tokens": 7126387.0,
+      "reward": 0.47137731313705444,
+      "reward_std": 0.20232117176055908,
+      "rewards/true_env_reward_fn/mean": 0.47137728333473206,
+      "rewards/true_env_reward_fn/std": 0.20232117176055908,
       "step": 289,
-      "step_time": 11.744046860998424
+      "step_time": 10.741382757999418
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8108,26 +8108,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.264986515045166,
-      "epoch": 2.3577235772357725,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001335782726528123,
-      "kl": 1.7416054106433876e-05,
-      "learning_rate": 1.0924301926274248e-07,
-      "loss": 8.602528396295384e-07,
-      "num_tokens": 1187311.0,
-      "reward": 0.4899469017982483,
-      "reward_std": 0.24049179255962372,
-      "rewards/true_env_reward_fn/mean": 0.4899469017982483,
-      "rewards/true_env_reward_fn/std": 0.24049179255962372,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 65.0,
+      "completions/mean_terminated_length": 65.0,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2131870985031128,
+      "epoch": 7.073170731707317,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07972414791584015,
+      "kl": 0.00010809541072376305,
+      "learning_rate": 2.426634296726955e-07,
+      "loss": 0.07707639783620834,
+      "num_tokens": 7150947.0,
+      "reward": 0.4951697587966919,
+      "reward_std": 0.31705158948898315,
+      "rewards/true_env_reward_fn/mean": 0.4951697587966919,
+      "rewards/true_env_reward_fn/std": 0.31705158948898315,
       "step": 290,
-      "step_time": 4.003069795000556
+      "step_time": 11.908707627000695
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8136,26 +8136,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 65.875,
-      "completions/mean_terminated_length": 65.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.222926139831543,
-      "epoch": 2.3658536585365852,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.209110481198877e-05,
-      "kl": 1.1452370472397888e-05,
-      "learning_rate": 1.0665668633705572e-07,
-      "loss": 5.764911747974111e-07,
-      "num_tokens": 1191370.0,
-      "reward": 0.5930472612380981,
-      "reward_std": 0.14818456768989563,
-      "rewards/true_env_reward_fn/mean": 0.5930472612380981,
-      "rewards/true_env_reward_fn/std": 0.14818456768989563,
+      "completions/max_length": 144.0,
+      "completions/max_terminated_length": 144.0,
+      "completions/mean_length": 66.41667175292969,
+      "completions/mean_terminated_length": 66.41667175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2247934639453888,
+      "epoch": 7.097560975609756,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06208934634923935,
+      "kl": 8.92497146196547e-05,
+      "learning_rate": 2.390229912157033e-07,
+      "loss": 0.021278446540236473,
+      "num_tokens": 7174671.0,
+      "reward": 0.44533461332321167,
+      "reward_std": 0.20755091309547424,
+      "rewards/true_env_reward_fn/mean": 0.4453345835208893,
+      "rewards/true_env_reward_fn/std": 0.20755092799663544,
       "step": 291,
-      "step_time": 4.77749846199913
+      "step_time": 11.859711304000484
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8164,26 +8164,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 65.625,
-      "completions/mean_terminated_length": 65.625,
-      "completions/min_length": 56.0,
-      "completions/min_terminated_length": 56.0,
-      "entropy": 1.313057780265808,
-      "epoch": 2.3739837398373984,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.671779556199908e-05,
-      "kl": 1.1384066056052689e-05,
-      "learning_rate": 1.0409661956807174e-07,
-      "loss": 5.689008162335085e-07,
-      "num_tokens": 1195851.0,
-      "reward": 0.49959999322891235,
-      "reward_std": 0.02822280302643776,
-      "rewards/true_env_reward_fn/mean": 0.49959999322891235,
-      "rewards/true_env_reward_fn/std": 0.02822280302643776,
+      "completions/max_length": 110.0,
+      "completions/max_terminated_length": 110.0,
+      "completions/mean_length": 66.25,
+      "completions/mean_terminated_length": 66.25,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.3344177305698395,
+      "epoch": 7.121951219512195,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08249509334564209,
+      "kl": 0.00013599474277725676,
+      "learning_rate": 2.3540146951369966e-07,
+      "loss": 0.05746981129050255,
+      "num_tokens": 7202291.0,
+      "reward": 0.36803489923477173,
+      "reward_std": 0.22159868478775024,
+      "rewards/true_env_reward_fn/mean": 0.36803486943244934,
+      "rewards/true_env_reward_fn/std": 0.22159868478775024,
       "step": 292,
-      "step_time": 3.4948791150000034
+      "step_time": 9.794866193998132
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8192,26 +8192,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.3050158619880676,
-      "epoch": 2.3821138211382116,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15694627165794373,
-      "kl": 2.6891467314271722e-05,
-      "learning_rate": 1.0156304818588308e-07,
-      "loss": -0.044491954147815704,
-      "num_tokens": 1201316.0,
-      "reward": 0.18382371962070465,
-      "reward_std": 0.18414245545864105,
-      "rewards/true_env_reward_fn/mean": 0.18382371962070465,
-      "rewards/true_env_reward_fn/std": 0.18414245545864105,
+      "completions/max_length": 172.0,
+      "completions/max_terminated_length": 172.0,
+      "completions/mean_length": 64.02083587646484,
+      "completions/mean_terminated_length": 64.02083587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2605153024196625,
+      "epoch": 7.146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07226194441318512,
+      "kl": 7.86567807153915e-05,
+      "learning_rate": 2.3179912707041666e-07,
+      "loss": -0.06701754778623581,
+      "num_tokens": 7221060.0,
+      "reward": 0.6259939670562744,
+      "reward_std": 0.30331701040267944,
+      "rewards/true_env_reward_fn/mean": 0.6259939074516296,
+      "rewards/true_env_reward_fn/std": 0.30331701040267944,
       "step": 293,
-      "step_time": 3.900356202000694
+      "step_time": 13.536596455999643
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8220,26 +8220,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 52.25,
-      "completions/mean_terminated_length": 52.25,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2079132199287415,
-      "epoch": 2.3902439024390243,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.25304585695266724,
-      "kl": 4.250183792464668e-05,
-      "learning_rate": 9.905619904816749e-08,
-      "loss": 0.008140146732330322,
-      "num_tokens": 1204090.0,
-      "reward": 0.43626630306243896,
-      "reward_std": 0.3027261197566986,
-      "rewards/true_env_reward_fn/mean": 0.43626630306243896,
-      "rewards/true_env_reward_fn/std": 0.3027261197566986,
+      "completions/max_length": 240.0,
+      "completions/max_terminated_length": 240.0,
+      "completions/mean_length": 76.83333587646484,
+      "completions/mean_terminated_length": 76.83333587646484,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.275952398777008,
+      "epoch": 7.170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07967559248209,
+      "kl": 0.0001376894815621199,
+      "learning_rate": 2.2821622499938948e-07,
+      "loss": -0.024285754188895226,
+      "num_tokens": 7251156.0,
+      "reward": 0.3353421688079834,
+      "reward_std": 0.26816248893737793,
+      "rewards/true_env_reward_fn/mean": 0.3353421688079834,
+      "rewards/true_env_reward_fn/std": 0.26816248893737793,
       "step": 294,
-      "step_time": 3.1466946830005327
+      "step_time": 19.762229363001097
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8248,26 +8248,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 64.375,
-      "completions/mean_terminated_length": 64.375,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2324314713478088,
-      "epoch": 2.3983739837398375,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011851672024931759,
-      "kl": 1.4868088328512385e-05,
-      "learning_rate": 9.657629661987531e-08,
-      "loss": 7.534490578109398e-07,
-      "num_tokens": 1206449.0,
-      "reward": 0.7232838273048401,
-      "reward_std": 0.008955853059887886,
-      "rewards/true_env_reward_fn/mean": 0.7232838273048401,
-      "rewards/true_env_reward_fn/std": 0.008955853059887886,
+      "completions/max_length": 137.0,
+      "completions/max_terminated_length": 137.0,
+      "completions/mean_length": 64.60417175292969,
+      "completions/mean_terminated_length": 64.60417175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.4140338003635406,
+      "epoch": 7.195121951219512,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07337357848882675,
+      "kl": 0.00012654263991862535,
+      "learning_rate": 2.2465302300503008e-07,
+      "loss": -0.04723845049738884,
+      "num_tokens": 7277361.0,
+      "reward": 0.4155184030532837,
+      "reward_std": 0.23990045487880707,
+      "rewards/true_env_reward_fn/mean": 0.4155184030532837,
+      "rewards/true_env_reward_fn/std": 0.23990046977996826,
       "step": 295,
-      "step_time": 3.5492840760016406
+      "step_time": 11.263231479000751
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8276,26 +8276,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 65.125,
-      "completions/mean_terminated_length": 65.125,
-      "completions/min_length": 57.0,
-      "completions/min_terminated_length": 57.0,
-      "entropy": 1.2110244631767273,
-      "epoch": 2.40650406504065,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 63.9375,
+      "completions/mean_terminated_length": 63.9375,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.22734934091568,
+      "epoch": 7.219512195121951,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18513579666614532,
-      "kl": 6.467000457632821e-05,
-      "learning_rate": 9.412356295313019e-08,
-      "loss": -0.008508354425430298,
-      "num_tokens": 1211294.0,
-      "reward": 0.38126999139785767,
-      "reward_std": 0.22339044511318207,
-      "rewards/true_env_reward_fn/mean": 0.38126999139785767,
-      "rewards/true_env_reward_fn/std": 0.22339043021202087,
+      "grad_norm": 0.0908711850643158,
+      "kl": 0.00016623977535346057,
+      "learning_rate": 2.2110977936380287e-07,
+      "loss": -0.03445049002766609,
+      "num_tokens": 7298998.0,
+      "reward": 0.564518392086029,
+      "reward_std": 0.30392351746559143,
+      "rewards/true_env_reward_fn/mean": 0.564518392086029,
+      "rewards/true_env_reward_fn/std": 0.30392348766326904,
       "step": 296,
-      "step_time": 3.6129159619995335
+      "step_time": 10.9702629049998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8304,26 +8304,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 58.75,
-      "completions/mean_terminated_length": 58.75,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.1828523874282837,
-      "epoch": 2.4146341463414633,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13915051519870758,
-      "kl": 2.76857699645916e-05,
-      "learning_rate": 9.169821766734668e-08,
-      "loss": -0.019797056913375854,
-      "num_tokens": 1214972.0,
-      "reward": 0.5778937339782715,
-      "reward_std": 0.4268997013568878,
-      "rewards/true_env_reward_fn/mean": 0.5778937339782715,
-      "rewards/true_env_reward_fn/std": 0.4268997013568878,
+      "completions/max_length": 156.0,
+      "completions/max_terminated_length": 156.0,
+      "completions/mean_length": 66.8125,
+      "completions/mean_terminated_length": 66.8125,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.312496393918991,
+      "epoch": 7.2439024390243905,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08918203413486481,
+      "kl": 0.00018221777827420738,
+      "learning_rate": 2.1758675090550328e-07,
+      "loss": 0.023428799584507942,
+      "num_tokens": 7320725.0,
+      "reward": 0.4695018231868744,
+      "reward_std": 0.2527172565460205,
+      "rewards/true_env_reward_fn/mean": 0.4695018231868744,
+      "rewards/true_env_reward_fn/std": 0.2527172565460205,
       "step": 297,
-      "step_time": 3.5299333029997797
+      "step_time": 12.689384352000161
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8332,26 +8332,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 61.125,
-      "completions/mean_terminated_length": 61.125,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2796242237091064,
-      "epoch": 2.4227642276422765,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23225097358226776,
-      "kl": 5.458398300106637e-05,
-      "learning_rate": 8.930047792956585e-08,
-      "loss": 0.028596192598342896,
-      "num_tokens": 1221117.0,
-      "reward": 0.09695600718259811,
-      "reward_std": 0.23755072057247162,
-      "rewards/true_env_reward_fn/mean": 0.09695600718259811,
-      "rewards/true_env_reward_fn/std": 0.23755072057247162,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 62.0,
+      "completions/mean_terminated_length": 62.0,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.260593295097351,
+      "epoch": 7.2682926829268295,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07724963873624802,
+      "kl": 4.775456409333856e-05,
+      "learning_rate": 2.1408419299464242e-07,
+      "loss": 0.03472111374139786,
+      "num_tokens": 7345397.0,
+      "reward": 0.39238378405570984,
+      "reward_std": 0.23648974299430847,
+      "rewards/true_env_reward_fn/mean": 0.39238378405570984,
+      "rewards/true_env_reward_fn/std": 0.23648977279663086,
       "step": 298,
-      "step_time": 3.9654863289997593
+      "step_time": 10.398283558999537
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8360,26 +8360,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.3131609559059143,
-      "epoch": 2.430894308943089,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15491950511932373,
-      "kl": 2.3678386241954286e-05,
-      "learning_rate": 8.693055843500867e-08,
-      "loss": 0.08264091610908508,
-      "num_tokens": 1226670.0,
-      "reward": 0.22746901214122772,
-      "reward_std": 0.2765822112560272,
-      "rewards/true_env_reward_fn/mean": 0.22746901214122772,
-      "rewards/true_env_reward_fn/std": 0.2765822112560272,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 75.375,
+      "completions/mean_terminated_length": 75.375,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.272550791501999,
+      "epoch": 7.2926829268292686,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0737687349319458,
+      "kl": 7.711273838140187e-05,
+      "learning_rate": 2.1060235951193578e-07,
+      "loss": 0.010874807834625244,
+      "num_tokens": 7371231.0,
+      "reward": 0.4486481845378876,
+      "reward_std": 0.26590272784233093,
+      "rewards/true_env_reward_fn/mean": 0.4486481845378876,
+      "rewards/true_env_reward_fn/std": 0.26590269804000854,
       "step": 299,
-      "step_time": 3.9300464680000005
+      "step_time": 12.961759718001304
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8388,26 +8388,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 61.75,
-      "completions/mean_terminated_length": 61.75,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.113481342792511,
-      "epoch": 2.4390243902439024,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14261895418167114,
-      "kl": 2.8488028874562588e-05,
-      "learning_rate": 8.458867138785369e-08,
-      "loss": 0.012870386242866516,
-      "num_tokens": 1230460.0,
-      "reward": 0.6178936958312988,
-      "reward_std": 0.3830615282058716,
-      "rewards/true_env_reward_fn/mean": 0.6178936958312988,
-      "rewards/true_env_reward_fn/std": 0.3830614984035492,
+      "completions/max_length": 178.0,
+      "completions/max_terminated_length": 178.0,
+      "completions/mean_length": 64.5,
+      "completions/mean_terminated_length": 64.5,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3426357805728912,
+      "epoch": 7.317073170731708,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09169661998748779,
+      "kl": 0.00015296797573682852,
+      "learning_rate": 2.071415028359026e-07,
+      "loss": -0.0708877444267273,
+      "num_tokens": 7397151.0,
+      "reward": 0.5012298822402954,
+      "reward_std": 0.21363919973373413,
+      "rewards/true_env_reward_fn/mean": 0.5012298226356506,
+      "rewards/true_env_reward_fn/std": 0.21363921463489532,
       "step": 300,
-      "step_time": 4.004705740000645
+      "step_time": 13.761256955000135
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8416,26 +8416,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 54.25,
-      "completions/mean_terminated_length": 54.25,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2633118629455566,
-      "epoch": 2.4471544715447155,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1559160053730011,
-      "kl": 3.0400691230170196e-05,
-      "learning_rate": 8.227502648223494e-08,
-      "loss": 0.023099511861801147,
-      "num_tokens": 1234850.0,
-      "reward": 0.3328326642513275,
-      "reward_std": 0.2606535255908966,
-      "rewards/true_env_reward_fn/mean": 0.3328326642513275,
-      "rewards/true_env_reward_fn/std": 0.260653555393219,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 70.75,
+      "completions/mean_terminated_length": 70.75,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2883423566818237,
+      "epoch": 7.341463414634147,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06652592122554779,
+      "kl": 0.00011371383516234346,
+      "learning_rate": 2.0370187382457066e-07,
+      "loss": -0.011642830446362495,
+      "num_tokens": 7426403.0,
+      "reward": 0.3339906632900238,
+      "reward_std": 0.20955638587474823,
+      "rewards/true_env_reward_fn/mean": 0.3339906632900238,
+      "rewards/true_env_reward_fn/std": 0.20955640077590942,
       "step": 301,
-      "step_time": 4.035395368000536
+      "step_time": 9.628323140000248
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8444,26 +8444,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 154.0,
-      "completions/max_terminated_length": 154.0,
-      "completions/mean_length": 79.875,
-      "completions/mean_terminated_length": 79.875,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.4545465111732483,
-      "epoch": 2.4552845528455283,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13911883533000946,
-      "kl": 5.561073703574948e-05,
-      "learning_rate": 7.99898308834662e-08,
-      "loss": 0.10435273498296738,
-      "num_tokens": 1237629.0,
-      "reward": 0.3929310441017151,
-      "reward_std": 0.294041246175766,
-      "rewards/true_env_reward_fn/mean": 0.3929310441017151,
-      "rewards/true_env_reward_fn/std": 0.294041246175766,
+      "completions/max_length": 152.0,
+      "completions/max_terminated_length": 152.0,
+      "completions/mean_length": 68.02083587646484,
+      "completions/mean_terminated_length": 68.02083587646484,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 1.1677636206150055,
+      "epoch": 7.365853658536586,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08921164274215698,
+      "kl": 7.750577606202569e-05,
+      "learning_rate": 2.0028372179729402e-07,
+      "loss": 0.024114318192005157,
+      "num_tokens": 7458380.0,
+      "reward": 0.3653101921081543,
+      "reward_std": 0.3042241334915161,
+      "rewards/true_env_reward_fn/mean": 0.3653101921081543,
+      "rewards/true_env_reward_fn/std": 0.3042241334915161,
       "step": 302,
-      "step_time": 6.382147416999942
+      "step_time": 17.152215452000746
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8472,26 +8472,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 61.625,
-      "completions/mean_terminated_length": 61.625,
-      "completions/min_length": 28.0,
-      "completions/min_terminated_length": 28.0,
-      "entropy": 1.4197523593902588,
-      "epoch": 2.4634146341463414,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11274738609790802,
-      "kl": 1.6755700926296413e-05,
-      "learning_rate": 7.773328920949151e-08,
-      "loss": -0.1040230244398117,
-      "num_tokens": 1240738.0,
-      "reward": 0.6045305728912354,
-      "reward_std": 0.10337947309017181,
-      "rewards/true_env_reward_fn/mean": 0.6045305728912354,
-      "rewards/true_env_reward_fn/std": 0.10337948054075241,
+      "completions/max_length": 112.0,
+      "completions/max_terminated_length": 112.0,
+      "completions/mean_length": 61.04166793823242,
+      "completions/mean_terminated_length": 61.04166793823242,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.1981053352355957,
+      "epoch": 7.390243902439025,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08189807087182999,
+      "kl": 0.00016033334395615384,
+      "learning_rate": 1.9688729451668111e-07,
+      "loss": 0.017046045511960983,
+      "num_tokens": 7481590.0,
+      "reward": 0.5003601312637329,
+      "reward_std": 0.3917968273162842,
+      "rewards/true_env_reward_fn/mean": 0.5003601312637329,
+      "rewards/true_env_reward_fn/std": 0.3917968273162842,
       "step": 303,
-      "step_time": 3.5754013399982796
+      "step_time": 10.428820308000468
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8500,26 +8500,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 105.0,
-      "completions/max_terminated_length": 105.0,
-      "completions/mean_length": 69.625,
-      "completions/mean_terminated_length": 69.625,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.3458821177482605,
-      "epoch": 2.4715447154471546,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11526551097631454,
-      "kl": 2.5664294298621826e-05,
-      "learning_rate": 7.550560351256309e-08,
-      "loss": 0.0017936527729034424,
-      "num_tokens": 1245803.0,
-      "reward": 0.2295326590538025,
-      "reward_std": 0.3014034032821655,
-      "rewards/true_env_reward_fn/mean": 0.2295326590538025,
-      "rewards/true_env_reward_fn/std": 0.3014034032821655,
+      "completions/max_length": 136.0,
+      "completions/max_terminated_length": 136.0,
+      "completions/mean_length": 64.95833587646484,
+      "completions/mean_terminated_length": 64.95833587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2136133313179016,
+      "epoch": 7.414634146341464,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07148941606283188,
+      "kl": 5.305510785547085e-05,
+      "learning_rate": 1.9351283817063546e-07,
+      "loss": -0.009052902460098267,
+      "num_tokens": 7508932.0,
+      "reward": 0.3940971791744232,
+      "reward_std": 0.2546152174472809,
+      "rewards/true_env_reward_fn/mean": 0.3940971791744232,
+      "rewards/true_env_reward_fn/std": 0.25461524724960327,
       "step": 304,
-      "step_time": 4.8661928239998815
+      "step_time": 12.12407543900099
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8528,26 +8528,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 66.125,
-      "completions/mean_terminated_length": 66.125,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2587900161743164,
-      "epoch": 2.4796747967479673,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1804642677307129,
-      "kl": 3.644101343525108e-05,
-      "learning_rate": 7.330697326114972e-08,
-      "loss": 0.006622403860092163,
-      "num_tokens": 1250224.0,
-      "reward": 0.392159104347229,
-      "reward_std": 0.23998720943927765,
-      "rewards/true_env_reward_fn/mean": 0.392159104347229,
-      "rewards/true_env_reward_fn/std": 0.23998722434043884,
+      "completions/max_length": 147.0,
+      "completions/max_terminated_length": 147.0,
+      "completions/mean_length": 61.875,
+      "completions/mean_terminated_length": 61.875,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2293521761894226,
+      "epoch": 7.439024390243903,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05877150595188141,
+      "kl": 0.00012415168203006033,
+      "learning_rate": 1.9016059735451156e-07,
+      "loss": -0.008430279791355133,
+      "num_tokens": 7535518.0,
+      "reward": 0.47240138053894043,
+      "reward_std": 0.2845028340816498,
+      "rewards/true_env_reward_fn/mean": 0.47240138053894043,
+      "rewards/true_env_reward_fn/std": 0.2845028340816498,
       "step": 305,
-      "step_time": 4.381737805999364
+      "step_time": 12.694503639000686
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8556,26 +8556,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 154.0,
-      "completions/max_terminated_length": 154.0,
-      "completions/mean_length": 88.75,
-      "completions/mean_terminated_length": 88.75,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.235496699810028,
-      "epoch": 2.4878048780487805,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010392541298642755,
-      "kl": 1.9250182049290743e-05,
-      "learning_rate": 7.113759532207599e-08,
-      "loss": 9.166102472590865e-07,
-      "num_tokens": 1256018.0,
-      "reward": 0.45198333263397217,
-      "reward_std": 0.0026191486977040768,
-      "rewards/true_env_reward_fn/mean": 0.45198333263397217,
-      "rewards/true_env_reward_fn/std": 0.0026191489305347204,
+      "completions/max_length": 240.0,
+      "completions/max_terminated_length": 240.0,
+      "completions/mean_length": 61.270835876464844,
+      "completions/mean_terminated_length": 61.270835876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.1673714816570282,
+      "epoch": 7.463414634146342,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07308074831962585,
+      "kl": 0.0001147145967479446,
+      "learning_rate": 1.8683081505338465e-07,
+      "loss": -0.06511729210615158,
+      "num_tokens": 7558883.0,
+      "reward": 0.4651026129722595,
+      "reward_std": 0.19122423231601715,
+      "rewards/true_env_reward_fn/mean": 0.46510258316993713,
+      "rewards/true_env_reward_fn/std": 0.19122423231601715,
       "step": 306,
-      "step_time": 6.6232522029986285
+      "step_time": 15.502204728000834
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8584,26 +8584,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.1592004895210266,
-      "epoch": 2.4959349593495936,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.044789915904403e-05,
-      "kl": 9.38343055167934e-06,
-      "learning_rate": 6.899766394289516e-08,
-      "loss": 4.709004315373022e-07,
-      "num_tokens": 1259980.0,
-      "reward": 0.4911326766014099,
-      "reward_std": 0.01917082816362381,
-      "rewards/true_env_reward_fn/mean": 0.4911326766014099,
-      "rewards/true_env_reward_fn/std": 0.01917083002626896,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 63.6875,
+      "completions/mean_terminated_length": 63.6875,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.304062157869339,
+      "epoch": 7.487804878048781,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0670589879155159,
+      "kl": 9.212431177729741e-05,
+      "learning_rate": 1.8352373262443915e-07,
+      "loss": -0.026926759630441666,
+      "num_tokens": 7578916.0,
+      "reward": 0.5716189742088318,
+      "reward_std": 0.19298586249351501,
+      "rewards/true_env_reward_fn/mean": 0.5716189742088318,
+      "rewards/true_env_reward_fn/std": 0.19298586249351501,
       "step": 307,
-      "step_time": 3.398790989002009
+      "step_time": 8.911180752998916
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8612,26 +8612,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 66.0,
-      "completions/max_terminated_length": 66.0,
-      "completions/mean_length": 53.625,
-      "completions/mean_terminated_length": 53.625,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.1962818503379822,
-      "epoch": 2.5040650406504064,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12975044548511505,
-      "kl": 9.622429843147984e-06,
-      "learning_rate": 6.688737073449563e-08,
-      "loss": -0.028225116431713104,
-      "num_tokens": 1265125.0,
-      "reward": 0.3520139753818512,
-      "reward_std": 0.4090423583984375,
-      "rewards/true_env_reward_fn/mean": 0.3520139753818512,
-      "rewards/true_env_reward_fn/std": 0.4090423583984375,
+      "completions/max_length": 144.0,
+      "completions/max_terminated_length": 144.0,
+      "completions/mean_length": 68.33333587646484,
+      "completions/mean_terminated_length": 68.33333587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2283632457256317,
+      "epoch": 7.512195121951219,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06950085610151291,
+      "kl": 9.545813054501195e-05,
+      "learning_rate": 1.80239589779473e-07,
+      "loss": 0.010137543082237244,
+      "num_tokens": 7604028.0,
+      "reward": 0.5349916219711304,
+      "reward_std": 0.2091754674911499,
+      "rewards/true_env_reward_fn/mean": 0.5349915623664856,
+      "rewards/true_env_reward_fn/std": 0.2091754525899887,
       "step": 308,
-      "step_time": 3.4128740100004507
+      "step_time": 11.745030509000571
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8640,26 +8640,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 54.75,
-      "completions/mean_terminated_length": 54.75,
+      "completions/max_length": 176.0,
+      "completions/max_terminated_length": 176.0,
+      "completions/mean_length": 77.5,
+      "completions/mean_terminated_length": 77.5,
       "completions/min_length": 37.0,
       "completions/min_terminated_length": 37.0,
-      "entropy": 1.1477364301681519,
-      "epoch": 2.5121951219512195,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.428078242810443e-05,
-      "kl": 1.2252480246388586e-05,
-      "learning_rate": 6.480690465394398e-08,
-      "loss": 6.147511157905683e-07,
-      "num_tokens": 1269679.0,
-      "reward": 0.4544333219528198,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.4544333219528198,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "entropy": 1.3315171897411346,
+      "epoch": 7.536585365853659,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05699850618839264,
+      "kl": 6.70248282403918e-05,
+      "learning_rate": 1.7697862456752271e-07,
+      "loss": 0.06499005854129791,
+      "num_tokens": 7628884.0,
+      "reward": 0.4583873748779297,
+      "reward_std": 0.26091766357421875,
+      "rewards/true_env_reward_fn/mean": 0.4583873748779297,
+      "rewards/true_env_reward_fn/std": 0.26091763377189636,
       "step": 309,
-      "step_time": 3.4761773770005675
+      "step_time": 13.87453935099984
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8668,26 +8668,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 60.625,
-      "completions/mean_terminated_length": 60.625,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.1964089274406433,
-      "epoch": 2.5203252032520327,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010435594595037401,
-      "kl": 1.349770900560543e-05,
-      "learning_rate": 6.275645198756629e-08,
-      "loss": 6.71000861984794e-07,
-      "num_tokens": 1272836.0,
-      "reward": 0.4899469017982483,
-      "reward_std": 0.24049179255962372,
-      "rewards/true_env_reward_fn/mean": 0.4899469017982483,
-      "rewards/true_env_reward_fn/std": 0.24049179255962372,
+      "completions/max_length": 139.0,
+      "completions/max_terminated_length": 139.0,
+      "completions/mean_length": 65.22917175292969,
+      "completions/mean_terminated_length": 65.22917175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2704036831855774,
+      "epoch": 7.560975609756097,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09379290789365768,
+      "kl": 0.0001983325491892174,
+      "learning_rate": 1.7374107335760934e-07,
+      "loss": 0.09065254032611847,
+      "num_tokens": 7651991.0,
+      "reward": 0.5210780501365662,
+      "reward_std": 0.2745552361011505,
+      "rewards/true_env_reward_fn/mean": 0.5210780501365662,
+      "rewards/true_env_reward_fn/std": 0.2745552361011505,
       "step": 310,
-      "step_time": 3.522990450999714
+      "step_time": 11.07200519900016
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8696,26 +8696,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 399.0,
-      "completions/max_terminated_length": 399.0,
-      "completions/mean_length": 101.125,
-      "completions/mean_terminated_length": 101.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.428375780582428,
-      "epoch": 2.5284552845528454,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10063836723566055,
-      "kl": 2.3114703708415618e-05,
-      "learning_rate": 6.073619633426675e-08,
-      "loss": -0.1970823109149933,
-      "num_tokens": 1279065.0,
-      "reward": 0.4598180055618286,
-      "reward_std": 0.23299095034599304,
-      "rewards/true_env_reward_fn/mean": 0.4598180055618286,
-      "rewards/true_env_reward_fn/std": 0.23299095034599304,
+      "completions/max_length": 166.0,
+      "completions/max_terminated_length": 166.0,
+      "completions/mean_length": 73.60417175292969,
+      "completions/mean_terminated_length": 73.60417175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2427658140659332,
+      "epoch": 7.585365853658536,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06011087819933891,
+      "kl": 6.545234282384627e-05,
+      "learning_rate": 1.7052717082160344e-07,
+      "loss": -0.0115435142070055,
+      "num_tokens": 7682948.0,
+      "reward": 0.3796111047267914,
+      "reward_std": 0.2212861329317093,
+      "rewards/true_env_reward_fn/mean": 0.3796111047267914,
+      "rewards/true_env_reward_fn/std": 0.22128616273403168,
       "step": 311,
-      "step_time": 16.64716850199875
+      "step_time": 12.894371897999918
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8724,26 +8724,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 62.875,
-      "completions/mean_terminated_length": 62.875,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.0843100249767303,
-      "epoch": 2.5365853658536586,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00015719492512289435,
-      "kl": 1.4557146187144099e-05,
-      "learning_rate": 5.8746318589089337e-08,
-      "loss": 7.280061709025176e-07,
-      "num_tokens": 1280592.0,
-      "reward": 0.8541955947875977,
-      "reward_std": 0.09160846471786499,
-      "rewards/true_env_reward_fn/mean": 0.8541955947875977,
-      "rewards/true_env_reward_fn/std": 0.09160846471786499,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 62.47916793823242,
+      "completions/mean_terminated_length": 62.47916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.1208404004573822,
+      "epoch": 7.609756097560975,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0799943059682846,
+      "kl": 0.0001426433700544294,
+      "learning_rate": 1.6733714991721738e-07,
+      "loss": -0.010216867551207542,
+      "num_tokens": 7704179.0,
+      "reward": 0.5524939298629761,
+      "reward_std": 0.26985135674476624,
+      "rewards/true_env_reward_fn/mean": 0.5524939298629761,
+      "rewards/true_env_reward_fn/std": 0.26985135674476624,
       "step": 312,
-      "step_time": 3.2888442910007143
+      "step_time": 8.69524126000033
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8752,26 +8752,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 72.0,
-      "completions/mean_terminated_length": 72.0,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.240959882736206,
-      "epoch": 2.5447154471544717,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12602119147777557,
-      "kl": 1.7604435015527997e-05,
-      "learning_rate": 5.67869969270196e-08,
-      "loss": 9.685754776000977e-07,
-      "num_tokens": 1284772.0,
-      "reward": 0.4882529377937317,
-      "reward_std": 0.2615475356578827,
-      "rewards/true_env_reward_fn/mean": 0.4882529377937317,
-      "rewards/true_env_reward_fn/std": 0.2615475356578827,
+      "completions/max_length": 121.0,
+      "completions/max_terminated_length": 121.0,
+      "completions/mean_length": 76.72917175292969,
+      "completions/mean_terminated_length": 76.72917175292969,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.2062768340110779,
+      "epoch": 7.634146341463414,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05652271583676338,
+      "kl": 0.00010435856393087306,
+      "learning_rate": 1.6417124187111774e-07,
+      "loss": 0.033915065228939056,
+      "num_tokens": 7732974.0,
+      "reward": 0.40553492307662964,
+      "reward_std": 0.2561950087547302,
+      "rewards/true_env_reward_fn/mean": 0.40553489327430725,
+      "rewards/true_env_reward_fn/std": 0.2561950087547302,
       "step": 313,
-      "step_time": 4.0729800409990276
+      "step_time": 11.456125995000548
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8780,26 +8780,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.1718184351921082,
-      "epoch": 2.5528455284552845,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14716969430446625,
-      "kl": 1.3087485967844259e-05,
-      "learning_rate": 5.4858406787030845e-08,
-      "loss": 0.016231566667556763,
-      "num_tokens": 1287839.0,
-      "reward": 0.6037359237670898,
-      "reward_std": 0.10266375541687012,
-      "rewards/true_env_reward_fn/mean": 0.6037359237670898,
-      "rewards/true_env_reward_fn/std": 0.10266375541687012,
+      "completions/max_length": 152.0,
+      "completions/max_terminated_length": 152.0,
+      "completions/mean_length": 63.41666793823242,
+      "completions/mean_terminated_length": 63.41666793823242,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 1.2864463925361633,
+      "epoch": 7.658536585365853,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08890142291784286,
+      "kl": 0.00014215287774277385,
+      "learning_rate": 1.6102967616216617e-07,
+      "loss": 0.04210362955927849,
+      "num_tokens": 7754650.0,
+      "reward": 0.5954334139823914,
+      "reward_std": 0.3237621784210205,
+      "rewards/true_env_reward_fn/mean": 0.5954334139823914,
+      "rewards/true_env_reward_fn/std": 0.3237621784210205,
       "step": 314,
-      "step_time": 3.175157601999672
+      "step_time": 11.561733381999147
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8808,26 +8808,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 52.0,
-      "completions/max_terminated_length": 52.0,
-      "completions/mean_length": 40.5,
-      "completions/mean_terminated_length": 40.5,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.3117390871047974,
-      "epoch": 2.5609756097560976,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2794886827468872,
-      "kl": 4.65317443740787e-05,
-      "learning_rate": 5.29607208563756e-08,
-      "loss": -0.09303665161132812,
-      "num_tokens": 1291515.0,
-      "reward": 0.5445280075073242,
-      "reward_std": 0.19415061175823212,
-      "rewards/true_env_reward_fn/mean": 0.5445280075073242,
-      "rewards/true_env_reward_fn/std": 0.19415059685707092,
+      "completions/max_length": 175.0,
+      "completions/max_terminated_length": 175.0,
+      "completions/mean_length": 69.22917175292969,
+      "completions/mean_terminated_length": 69.22917175292969,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2406023740768433,
+      "epoch": 7.682926829268292,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05483023077249527,
+      "kl": 3.3986466860369546e-05,
+      "learning_rate": 1.5791268050478483e-07,
+      "loss": -0.03052404522895813,
+      "num_tokens": 7777389.0,
+      "reward": 0.5392192602157593,
+      "reward_std": 0.21974749863147736,
+      "rewards/true_env_reward_fn/mean": 0.5392192006111145,
+      "rewards/true_env_reward_fn/std": 0.21974751353263855,
       "step": 315,
-      "step_time": 2.636586960999921
+      "step_time": 13.48241268899983
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8836,26 +8836,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 51.5,
-      "completions/mean_terminated_length": 51.5,
-      "completions/min_length": 13.0,
-      "completions/min_terminated_length": 13.0,
-      "entropy": 1.15413236618042,
-      "epoch": 2.569105691056911,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22290591895580292,
-      "kl": 6.188569841469871e-05,
-      "learning_rate": 5.10941090551225e-08,
-      "loss": -0.013920806348323822,
-      "num_tokens": 1296399.0,
-      "reward": 0.3418610095977783,
-      "reward_std": 0.29834023118019104,
-      "rewards/true_env_reward_fn/mean": 0.3418610095977783,
-      "rewards/true_env_reward_fn/std": 0.2983402609825134,
+      "completions/max_length": 94.0,
+      "completions/max_terminated_length": 94.0,
+      "completions/mean_length": 57.97916793823242,
+      "completions/mean_terminated_length": 57.97916793823242,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2167797982692719,
+      "epoch": 7.7073170731707314,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08746550232172012,
+      "kl": 0.00011399560207792092,
+      "learning_rate": 1.5482048083245114e-07,
+      "loss": 0.014903642237186432,
+      "num_tokens": 7807828.0,
+      "reward": 0.29947829246520996,
+      "reward_std": 0.2547810673713684,
+      "rewards/true_env_reward_fn/mean": 0.29947829246520996,
+      "rewards/true_env_reward_fn/std": 0.2547810673713684,
       "step": 316,
-      "step_time": 3.7106533750011295
+      "step_time": 10.080044547000398
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8864,26 +8864,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 265.0,
-      "completions/max_terminated_length": 265.0,
-      "completions/mean_length": 102.875,
-      "completions/mean_terminated_length": 102.875,
-      "completions/min_length": 57.0,
-      "completions/min_terminated_length": 57.0,
-      "entropy": 1.2713149785995483,
-      "epoch": 2.5772357723577235,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11319029331207275,
-      "kl": 1.3128932096151402e-05,
-      "learning_rate": 4.9258738520942025e-08,
-      "loss": 0.04848237335681915,
-      "num_tokens": 1301626.0,
-      "reward": 0.4802166819572449,
-      "reward_std": 0.05550921708345413,
-      "rewards/true_env_reward_fn/mean": 0.4802166819572449,
-      "rewards/true_env_reward_fn/std": 0.05550922453403473,
+      "completions/max_length": 147.0,
+      "completions/max_terminated_length": 147.0,
+      "completions/mean_length": 65.3125,
+      "completions/mean_terminated_length": 65.3125,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2823624312877655,
+      "epoch": 7.7317073170731705,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07846319675445557,
+      "kl": 0.00012852110194216948,
+      "learning_rate": 1.517533012813217e-07,
+      "loss": 0.05300665646791458,
+      "num_tokens": 7834227.0,
+      "reward": 0.4816446304321289,
+      "reward_std": 0.3394080102443695,
+      "rewards/true_env_reward_fn/mean": 0.4816446304321289,
+      "rewards/true_env_reward_fn/std": 0.3394079804420471,
       "step": 317,
-      "step_time": 11.140286670997739
+      "step_time": 12.856840839001052
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8892,26 +8892,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.0,
-      "completions/max_terminated_length": 64.0,
-      "completions/mean_length": 45.625,
-      "completions/mean_terminated_length": 45.625,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 0.9857950508594513,
-      "epoch": 2.5853658536585367,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011810586147475988,
-      "kl": 1.2108953342249151e-05,
-      "learning_rate": 4.745477359414045e-08,
-      "loss": 6.04832052886195e-07,
-      "num_tokens": 1303619.0,
-      "reward": 0.6204532384872437,
-      "reward_std": 0.10097470134496689,
-      "rewards/true_env_reward_fn/mean": 0.6204532384872437,
-      "rewards/true_env_reward_fn/std": 0.10097470879554749,
+      "completions/max_length": 264.0,
+      "completions/max_terminated_length": 264.0,
+      "completions/mean_length": 83.47917175292969,
+      "completions/mean_terminated_length": 83.47917175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.3714390099048615,
+      "epoch": 7.7560975609756095,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0722746029496193,
+      "kl": 0.00011485655522847082,
+      "learning_rate": 1.4871136417398405e-07,
+      "loss": -0.009422918781638145,
+      "num_tokens": 7860362.0,
+      "reward": 0.44833892583847046,
+      "reward_std": 0.24655339121818542,
+      "rewards/true_env_reward_fn/mean": 0.44833889603614807,
+      "rewards/true_env_reward_fn/std": 0.24655337631702423,
       "step": 318,
-      "step_time": 2.977002043000539
+      "step_time": 18.09142264499951
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8920,26 +8920,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2560631036758423,
-      "epoch": 2.59349593495935,
+      "completions/max_length": 203.0,
+      "completions/max_terminated_length": 203.0,
+      "completions/mean_length": 69.10417175292969,
+      "completions/mean_terminated_length": 69.10417175292969,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2680339813232422,
+      "epoch": 7.780487804878049,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24155104160308838,
-      "kl": 3.4935148505610414e-05,
-      "learning_rate": 4.5682375802945295e-08,
-      "loss": 0.1817030906677246,
-      "num_tokens": 1310682.0,
-      "reward": 0.055160000920295715,
-      "reward_std": 0.13755998015403748,
-      "rewards/true_env_reward_fn/mean": 0.055160000920295715,
-      "rewards/true_env_reward_fn/std": 0.13755998015403748,
+      "grad_norm": 0.09518000483512878,
+      "kl": 0.00010953140827041352,
+      "learning_rate": 1.4569489000334433e-07,
+      "loss": -0.09500816464424133,
+      "num_tokens": 7882799.0,
+      "reward": 0.4905685782432556,
+      "reward_std": 0.261008620262146,
+      "rewards/true_env_reward_fn/mean": 0.4905685484409332,
+      "rewards/true_env_reward_fn/std": 0.261008620262146,
       "step": 319,
-      "step_time": 4.712957282001298
+      "step_time": 17.2091521250004
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8948,26 +8948,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 102.0,
-      "completions/max_terminated_length": 102.0,
-      "completions/mean_length": 69.75,
-      "completions/mean_terminated_length": 69.75,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2313570380210876,
-      "epoch": 2.6016260162601625,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011636545968940482,
-      "kl": 1.566714945511194e-05,
-      "learning_rate": 4.394170384904137e-08,
-      "loss": 7.867492968216538e-07,
-      "num_tokens": 1315436.0,
-      "reward": 0.4662666618824005,
-      "reward_std": 0.007412042934447527,
-      "rewards/true_env_reward_fn/mean": 0.4662666618824005,
-      "rewards/true_env_reward_fn/std": 0.007412043400108814,
+      "completions/max_length": 257.0,
+      "completions/max_terminated_length": 257.0,
+      "completions/mean_length": 66.45833587646484,
+      "completions/mean_terminated_length": 66.45833587646484,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2107920348644257,
+      "epoch": 7.804878048780488,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07063736766576767,
+      "kl": 3.921870529666194e-05,
+      "learning_rate": 1.4270409741664268e-07,
+      "loss": -0.011502111330628395,
+      "num_tokens": 7901717.0,
+      "reward": 0.566825807094574,
+      "reward_std": 0.18909600377082825,
+      "rewards/true_env_reward_fn/mean": 0.566825807094574,
+      "rewards/true_env_reward_fn/std": 0.18909598886966705,
       "step": 320,
-      "step_time": 4.536037327998201
+      "step_time": 15.649325063000106
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -8976,26 +8976,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 49.625,
-      "completions/mean_terminated_length": 49.625,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.0671695470809937,
-      "epoch": 2.6097560975609757,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00013134608161635697,
-      "kl": 1.7747691344993655e-05,
-      "learning_rate": 4.223291359336074e-08,
-      "loss": 8.864202527547604e-07,
-      "num_tokens": 1318837.0,
-      "reward": 0.6651140451431274,
-      "reward_std": 0.20516473054885864,
-      "rewards/true_env_reward_fn/mean": 0.6651140451431274,
-      "rewards/true_env_reward_fn/std": 0.20516474545001984,
+      "completions/max_length": 106.0,
+      "completions/max_terminated_length": 106.0,
+      "completions/mean_length": 66.66667175292969,
+      "completions/mean_terminated_length": 66.66667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2565636038780212,
+      "epoch": 7.829268292682927,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06063119322061539,
+      "kl": 4.300068394513801e-05,
+      "learning_rate": 1.3973920319960652e-07,
+      "loss": 0.001966973766684532,
+      "num_tokens": 7919013.0,
+      "reward": 0.6115278005599976,
+      "reward_std": 0.17882205545902252,
+      "rewards/true_env_reward_fn/mean": 0.6115277409553528,
+      "rewards/true_env_reward_fn/std": 0.17882204055786133,
       "step": 321,
-      "step_time": 2.9819667609990574
+      "step_time": 7.359487544999865
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9004,26 +9004,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 178.0,
-      "completions/max_terminated_length": 178.0,
-      "completions/mean_length": 85.25,
-      "completions/mean_terminated_length": 85.25,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.3910706639289856,
-      "epoch": 2.617886178861789,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1727662831544876,
-      "kl": 1.9147763850924093e-05,
-      "learning_rate": 4.05561580421272e-08,
-      "loss": 0.07752666622400284,
-      "num_tokens": 1324515.0,
-      "reward": 0.43344998359680176,
-      "reward_std": 0.039401449263095856,
-      "rewards/true_env_reward_fn/mean": 0.43344998359680176,
-      "rewards/true_env_reward_fn/std": 0.039401449263095856,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 57.083335876464844,
+      "completions/mean_terminated_length": 57.083335876464844,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2200327813625336,
+      "epoch": 7.853658536585366,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07409472018480301,
+      "kl": 0.00011575021017051768,
+      "learning_rate": 1.368004222607355e-07,
+      "loss": -0.047135986387729645,
+      "num_tokens": 7941137.0,
+      "reward": 0.5057475566864014,
+      "reward_std": 0.3366250991821289,
+      "rewards/true_env_reward_fn/mean": 0.5057475566864014,
+      "rewards/true_env_reward_fn/std": 0.3366251289844513,
       "step": 322,
-      "step_time": 7.645187717998851
+      "step_time": 10.691665401999671
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9032,26 +9032,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 65.125,
-      "completions/mean_terminated_length": 65.125,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.1752581596374512,
-      "epoch": 2.6260162601626016,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14819224178791046,
-      "kl": 1.1951405213039834e-05,
-      "learning_rate": 3.891158733315505e-08,
-      "loss": 0.0940035805106163,
-      "num_tokens": 1327268.0,
-      "reward": 0.6361198425292969,
-      "reward_std": 0.13543039560317993,
-      "rewards/true_env_reward_fn/mean": 0.6361198425292969,
-      "rewards/true_env_reward_fn/std": 0.13543038070201874,
+      "completions/max_length": 100.0,
+      "completions/max_terminated_length": 100.0,
+      "completions/mean_length": 57.520835876464844,
+      "completions/mean_terminated_length": 57.520835876464844,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 1.3097383677959442,
+      "epoch": 7.878048780487805,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0950520932674408,
+      "kl": 0.00014910039135429543,
+      "learning_rate": 1.338879676157249e-07,
+      "loss": -0.018300604075193405,
+      "num_tokens": 7967298.0,
+      "reward": 0.39754772186279297,
+      "reward_std": 0.227029949426651,
+      "rewards/true_env_reward_fn/mean": 0.39754772186279297,
+      "rewards/true_env_reward_fn/std": 0.2270299643278122,
       "step": 323,
-      "step_time": 4.102753150000353
+      "step_time": 9.646710404997975
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9060,26 +9060,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 69.75,
-      "completions/mean_terminated_length": 69.75,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.4734079241752625,
-      "epoch": 2.6341463414634148,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21136029064655304,
-      "kl": 4.428037027537357e-05,
-      "learning_rate": 3.7299348722406965e-08,
-      "loss": -0.13074398040771484,
-      "num_tokens": 1331614.0,
-      "reward": 0.3051303029060364,
-      "reward_std": 0.20326493680477142,
-      "rewards/true_env_reward_fn/mean": 0.3051303029060364,
-      "rewards/true_env_reward_fn/std": 0.20326495170593262,
+      "completions/max_length": 165.0,
+      "completions/max_terminated_length": 165.0,
+      "completions/mean_length": 63.04166793823242,
+      "completions/mean_terminated_length": 63.04166793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3998334109783173,
+      "epoch": 7.902439024390244,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08187483251094818,
+      "kl": 0.0001343226194876479,
+      "learning_rate": 1.310020503720254e-07,
+      "loss": 0.10223554074764252,
+      "num_tokens": 7989452.0,
+      "reward": 0.4514659643173218,
+      "reward_std": 0.3259531557559967,
+      "rewards/true_env_reward_fn/mean": 0.4514659643173218,
+      "rewards/true_env_reward_fn/std": 0.3259531557559967,
       "step": 324,
-      "step_time": 5.413495869999679
+      "step_time": 13.284335969000495
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9088,26 +9088,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 179.0,
-      "completions/max_terminated_length": 179.0,
-      "completions/mean_length": 78.625,
-      "completions/mean_terminated_length": 78.625,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.5631850361824036,
-      "epoch": 2.642276422764228,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18741382658481598,
-      "kl": 3.721813300217036e-05,
-      "learning_rate": 3.57195865708074e-08,
-      "loss": 0.20410414040088654,
-      "num_tokens": 1334991.0,
-      "reward": 0.4402502775192261,
-      "reward_std": 0.27313899993896484,
-      "rewards/true_env_reward_fn/mean": 0.4402502775192261,
-      "rewards/true_env_reward_fn/std": 0.27313899993896484,
+      "completions/max_length": 248.0,
+      "completions/max_terminated_length": 248.0,
+      "completions/mean_length": 78.77083587646484,
+      "completions/mean_terminated_length": 78.77083587646484,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3267191052436829,
+      "epoch": 7.926829268292683,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06648680567741394,
+      "kl": 5.966442222415935e-05,
+      "learning_rate": 1.281428797135402e-07,
+      "loss": -0.046651843935251236,
+      "num_tokens": 8011065.0,
+      "reward": 0.6008384227752686,
+      "reward_std": 0.19184507429599762,
+      "rewards/true_env_reward_fn/mean": 0.6008384227752686,
+      "rewards/true_env_reward_fn/std": 0.19184507429599762,
       "step": 325,
-      "step_time": 7.439067015000546
+      "step_time": 18.06017050799983
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9116,26 +9116,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 58.875,
-      "completions/mean_terminated_length": 58.875,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2137504816055298,
-      "epoch": 2.6504065040650406,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1392536610364914,
-      "kl": 2.5514624212519266e-05,
-      "learning_rate": 3.417244233131709e-08,
-      "loss": -0.07728786766529083,
-      "num_tokens": 1340886.0,
-      "reward": 0.3032499849796295,
-      "reward_std": 0.24293993413448334,
-      "rewards/true_env_reward_fn/mean": 0.3032499849796295,
-      "rewards/true_env_reward_fn/std": 0.24293994903564453,
+      "completions/max_length": 108.0,
+      "completions/max_terminated_length": 108.0,
+      "completions/mean_length": 61.54166793823242,
+      "completions/mean_terminated_length": 61.54166793823242,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 1.4001933634281158,
+      "epoch": 7.951219512195122,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10428700596094131,
+      "kl": 0.0001954599174496252,
+      "learning_rate": 1.253106628854635e-07,
+      "loss": -0.08898806571960449,
+      "num_tokens": 8042035.0,
+      "reward": 0.21636998653411865,
+      "reward_std": 0.26617059111595154,
+      "rewards/true_env_reward_fn/mean": 0.21636998653411865,
+      "rewards/true_env_reward_fn/std": 0.26617059111595154,
       "step": 326,
-      "step_time": 3.6837621579998086
+      "step_time": 9.872497149000083
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9144,26 +9144,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 45.125,
-      "completions/mean_terminated_length": 45.125,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.1947113871574402,
-      "epoch": 2.658536585365854,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23312504589557648,
-      "kl": 6.829304766142741e-05,
-      "learning_rate": 3.265805453626713e-08,
-      "loss": 0.08558641374111176,
-      "num_tokens": 1344823.0,
-      "reward": 0.26869305968284607,
-      "reward_std": 0.30387282371520996,
-      "rewards/true_env_reward_fn/mean": 0.26869305968284607,
-      "rewards/true_env_reward_fn/std": 0.30387285351753235,
+      "completions/max_length": 118.0,
+      "completions/max_terminated_length": 118.0,
+      "completions/mean_length": 69.91667175292969,
+      "completions/mean_terminated_length": 69.91667175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.198440283536911,
+      "epoch": 7.975609756097561,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06512034684419632,
+      "kl": 9.893041169561911e-05,
+      "learning_rate": 1.2250560517925745e-07,
+      "loss": -0.004943478852510452,
+      "num_tokens": 8066991.0,
+      "reward": 0.5199041962623596,
+      "reward_std": 0.2916473150253296,
+      "rewards/true_env_reward_fn/mean": 0.5199041962623596,
+      "rewards/true_env_reward_fn/std": 0.2916473150253296,
       "step": 327,
-      "step_time": 3.26592919200084
+      "step_time": 13.27990607999891
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9172,26 +9172,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 70.125,
-      "completions/mean_terminated_length": 70.125,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1701073944568634,
-      "epoch": 2.6666666666666665,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12238985300064087,
-      "kl": 1.6267454157059547e-05,
-      "learning_rate": 3.117655878495458e-08,
-      "loss": -0.024053269997239113,
-      "num_tokens": 1348012.0,
-      "reward": 0.5990839004516602,
-      "reward_std": 0.21510076522827148,
-      "rewards/true_env_reward_fn/mean": 0.5990839004516602,
-      "rewards/true_env_reward_fn/std": 0.21510076522827148,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 60.875,
+      "completions/mean_terminated_length": 60.875,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.3655813038349152,
+      "epoch": 8.0,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08734066039323807,
+      "kl": 0.00011680843272188213,
+      "learning_rate": 1.197279099177731e-07,
+      "loss": -0.027742527425289154,
+      "num_tokens": 8095161.0,
+      "reward": 0.4403582811355591,
+      "reward_std": 0.20052418112754822,
+      "rewards/true_env_reward_fn/mean": 0.4403582811355591,
+      "rewards/true_env_reward_fn/std": 0.20052418112754822,
       "step": 328,
-      "step_time": 4.166060536001169
+      "step_time": 9.091917235000437
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9200,26 +9200,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 61.0,
-      "completions/max_terminated_length": 61.0,
-      "completions/mean_length": 49.75,
-      "completions/mean_terminated_length": 49.75,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.3588789701461792,
-      "epoch": 2.6747967479674797,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21684816479682922,
-      "kl": 0.00012067620264133438,
-      "learning_rate": 2.9728087731500884e-08,
-      "loss": 0.017311420291662216,
-      "num_tokens": 1352082.0,
-      "reward": 0.15680472552776337,
-      "reward_std": 0.3331984877586365,
-      "rewards/true_env_reward_fn/mean": 0.15680472552776337,
-      "rewards/true_env_reward_fn/std": 0.3331984877586365,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 63.645835876464844,
+      "completions/mean_terminated_length": 63.645835876464844,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.1942758560180664,
+      "epoch": 8.024390243902438,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07103214412927628,
+      "kl": 9.241796033165883e-05,
+      "learning_rate": 1.1697777844051104e-07,
+      "loss": -0.024517521262168884,
+      "num_tokens": 8117752.0,
+      "reward": 0.4961467981338501,
+      "reward_std": 0.2681204676628113,
+      "rewards/true_env_reward_fn/mean": 0.4961467981338501,
+      "rewards/true_env_reward_fn/std": 0.2681204676628113,
       "step": 329,
-      "step_time": 2.947034541000903
+      "step_time": 10.042522196999926
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9228,26 +9228,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 57.125,
-      "completions/mean_terminated_length": 57.125,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2207121849060059,
-      "epoch": 2.682926829268293,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14501330256462097,
-      "kl": 3.47198892995948e-05,
-      "learning_rate": 2.831277107297394e-08,
-      "loss": -0.009820610284805298,
-      "num_tokens": 1356867.0,
-      "reward": 0.508467972278595,
-      "reward_std": 0.10515833646059036,
-      "rewards/true_env_reward_fn/mean": 0.508467972278595,
-      "rewards/true_env_reward_fn/std": 0.10515833646059036,
+      "completions/max_length": 108.0,
+      "completions/max_terminated_length": 108.0,
+      "completions/mean_length": 71.58333587646484,
+      "completions/mean_terminated_length": 71.58333587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.209133356809616,
+      "epoch": 8.048780487804878,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06019354239106178,
+      "kl": 2.6403215088066645e-05,
+      "learning_rate": 1.142554100890285e-07,
+      "loss": -0.03160044550895691,
+      "num_tokens": 8140164.0,
+      "reward": 0.5655641555786133,
+      "reward_std": 0.1564219743013382,
+      "rewards/true_env_reward_fn/mean": 0.5655641555786133,
+      "rewards/true_env_reward_fn/std": 0.1564219743013382,
       "step": 330,
-      "step_time": 3.278030399000272
+      "step_time": 8.212663780001094
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9256,26 +9256,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 121.0,
-      "completions/max_terminated_length": 121.0,
-      "completions/mean_length": 71.5,
-      "completions/mean_terminated_length": 71.5,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2053698897361755,
-      "epoch": 2.6910569105691056,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1709476113319397,
-      "kl": 1.9107792468275875e-05,
-      "learning_rate": 2.693073553777494e-08,
-      "loss": 0.04886455833911896,
-      "num_tokens": 1361747.0,
-      "reward": 0.3092518746852875,
-      "reward_std": 0.40765801072120667,
-      "rewards/true_env_reward_fn/mean": 0.3092518746852875,
-      "rewards/true_env_reward_fn/std": 0.40765801072120667,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 62.47916793823242,
+      "completions/mean_terminated_length": 62.47916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.229485958814621,
+      "epoch": 8.073170731707316,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07769559323787689,
+      "kl": 0.0001446352634957293,
+      "learning_rate": 1.115610021924902e-07,
+      "loss": -0.01835213601589203,
+      "num_tokens": 8177099.0,
+      "reward": 0.1968289315700531,
+      "reward_std": 0.3009001910686493,
+      "rewards/true_env_reward_fn/mean": 0.1968289166688919,
+      "rewards/true_env_reward_fn/std": 0.3009001612663269,
       "step": 331,
-      "step_time": 5.705391269999382
+      "step_time": 16.508294159000798
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9284,26 +9284,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 70.625,
-      "completions/mean_terminated_length": 70.625,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.3019116520881653,
-      "epoch": 2.6991869918699187,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13118988275527954,
-      "kl": 6.723498518113047e-05,
-      "learning_rate": 2.5582104874290888e-08,
-      "loss": 0.04069610685110092,
-      "num_tokens": 1366792.0,
-      "reward": 0.39014729857444763,
-      "reward_std": 0.4048542380332947,
-      "rewards/true_env_reward_fn/mean": 0.39014729857444763,
-      "rewards/true_env_reward_fn/std": 0.40485426783561707,
+      "completions/max_length": 99.0,
+      "completions/max_terminated_length": 99.0,
+      "completions/mean_length": 55.72916793823242,
+      "completions/mean_terminated_length": 55.72916793823242,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.249097228050232,
+      "epoch": 8.097560975609756,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09750289469957352,
+      "kl": 0.00022061014351493213,
+      "learning_rate": 1.0889475005336446e-07,
+      "loss": 0.012823417782783508,
+      "num_tokens": 8202310.0,
+      "reward": 0.4960649907588959,
+      "reward_std": 0.28122010827064514,
+      "rewards/true_env_reward_fn/mean": 0.4960649907588959,
+      "rewards/true_env_reward_fn/std": 0.28122007846832275,
       "step": 332,
-      "step_time": 4.941442722001739
+      "step_time": 9.309349606999604
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9311,27 +9311,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
+      "completions/clipped_ratio": 0.02083333395421505,
       "completions/max_length": 512.0,
-      "completions/max_terminated_length": 129.0,
-      "completions/mean_length": 122.25,
-      "completions/mean_terminated_length": 66.5714340209961,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 0.8574695587158203,
-      "epoch": 2.7073170731707314,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.08299780637025833,
-      "kl": 8.204095138353296e-06,
-      "learning_rate": 2.4266999839814484e-08,
-      "loss": -0.5517755150794983,
-      "num_tokens": 1373066.0,
-      "reward": 0.19574999809265137,
-      "reward_std": 0.30308666825294495,
-      "rewards/true_env_reward_fn/mean": 0.19574999809265137,
-      "rewards/true_env_reward_fn/std": 0.30308666825294495,
+      "completions/max_terminated_length": 326.0,
+      "completions/mean_length": 82.35417175292969,
+      "completions/mean_terminated_length": 73.21276092529297,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3090683817863464,
+      "epoch": 8.121951219512194,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09015674889087677,
+      "kl": 0.00010220101285085548,
+      "learning_rate": 1.0625684693326725e-07,
+      "loss": 0.23226313292980194,
+      "num_tokens": 8227663.0,
+      "reward": 0.4449918270111084,
+      "reward_std": 0.24675849080085754,
+      "rewards/true_env_reward_fn/mean": 0.4449918270111084,
+      "rewards/true_env_reward_fn/std": 0.24675849080085754,
       "step": 333,
-      "step_time": 20.7688721859995
+      "step_time": 29.62071167800059
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9340,26 +9340,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 49.625,
-      "completions/mean_terminated_length": 49.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.191849946975708,
-      "epoch": 2.7154471544715446,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0001034519009408541,
-      "kl": 1.4263560842664447e-05,
-      "learning_rate": 2.2985538189730946e-08,
-      "loss": 7.244366315717343e-07,
-      "num_tokens": 1375815.0,
-      "reward": 0.6581807136535645,
-      "reward_std": 0.21257677674293518,
-      "rewards/true_env_reward_fn/mean": 0.6581807136535645,
-      "rewards/true_env_reward_fn/std": 0.21257677674293518,
+      "completions/max_length": 154.0,
+      "completions/max_terminated_length": 154.0,
+      "completions/mean_length": 67.52083587646484,
+      "completions/mean_terminated_length": 67.52083587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2623873353004456,
+      "epoch": 8.146341463414634,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09357088804244995,
+      "kl": 0.00020532654161797836,
+      "learning_rate": 1.0364748403895368e-07,
+      "loss": 0.015600509941577911,
+      "num_tokens": 8251736.0,
+      "reward": 0.49391698837280273,
+      "reward_std": 0.313683420419693,
+      "rewards/true_env_reward_fn/mean": 0.49391698837280273,
+      "rewards/true_env_reward_fn/std": 0.3136834502220154,
       "step": 334,
-      "step_time": 3.5656982629971026
+      "step_time": 14.77577510500032
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9368,26 +9368,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 59.75,
-      "completions/mean_terminated_length": 59.75,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.2259244322776794,
-      "epoch": 2.7235772357723578,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12885747849941254,
-      "kl": 4.455958514881786e-05,
-      "learning_rate": 2.1737834666974276e-08,
-      "loss": -0.06481382250785828,
-      "num_tokens": 1379933.0,
-      "reward": 0.31903696060180664,
-      "reward_std": 0.1877410113811493,
-      "rewards/true_env_reward_fn/mean": 0.31903696060180664,
-      "rewards/true_env_reward_fn/std": 0.18774102628231049,
+      "completions/max_length": 196.0,
+      "completions/max_terminated_length": 196.0,
+      "completions/mean_length": 89.25,
+      "completions/mean_terminated_length": 89.25,
+      "completions/min_length": 51.0,
+      "completions/min_terminated_length": 51.0,
+      "entropy": 1.3438844978809357,
+      "epoch": 8.170731707317072,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07505634427070618,
+      "kl": 8.66996942932019e-05,
+      "learning_rate": 1.0106685050845837e-07,
+      "loss": -0.006955280434340239,
+      "num_tokens": 8283596.0,
+      "reward": 0.37073618173599243,
+      "reward_std": 0.22015252709388733,
+      "rewards/true_env_reward_fn/mean": 0.37073615193367004,
+      "rewards/true_env_reward_fn/std": 0.22015254199504852,
       "step": 335,
-      "step_time": 3.712153099000716
+      "step_time": 14.388228956000603
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9396,26 +9396,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 63.0,
-      "completions/max_terminated_length": 63.0,
-      "completions/mean_length": 45.0,
-      "completions/mean_terminated_length": 45.0,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.2184038758277893,
-      "epoch": 2.7317073170731705,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.258829265832901,
-      "kl": 6.149329965410288e-05,
-      "learning_rate": 2.0524000991753465e-08,
-      "loss": -0.04611395671963692,
-      "num_tokens": 1383205.0,
-      "reward": 0.41517797112464905,
-      "reward_std": 0.33201757073402405,
-      "rewards/true_env_reward_fn/mean": 0.41517797112464905,
-      "rewards/true_env_reward_fn/std": 0.33201757073402405,
+      "completions/max_length": 196.0,
+      "completions/max_terminated_length": 196.0,
+      "completions/mean_length": 74.375,
+      "completions/mean_terminated_length": 74.375,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3619366586208344,
+      "epoch": 8.195121951219512,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07524604350328445,
+      "kl": 5.828110352013027e-05,
+      "learning_rate": 9.851513339738626e-08,
+      "loss": 0.02218097448348999,
+      "num_tokens": 8304798.0,
+      "reward": 0.6131325960159302,
+      "reward_std": 0.12095426768064499,
+      "rewards/true_env_reward_fn/mean": 0.6131325364112854,
+      "rewards/true_env_reward_fn/std": 0.12095426768064499,
       "step": 336,
-      "step_time": 3.02953305399933
+      "step_time": 12.145640587000344
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9423,27 +9423,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 206.0,
-      "completions/max_terminated_length": 206.0,
-      "completions/mean_length": 87.875,
-      "completions/mean_terminated_length": 87.875,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.2387716174125671,
-      "epoch": 2.7398373983739837,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16021011769771576,
-      "kl": 1.6921320366236614e-05,
-      "learning_rate": 1.9344145851548243e-08,
-      "loss": -0.14579075574874878,
-      "num_tokens": 1389136.0,
-      "reward": 0.4916413426399231,
-      "reward_std": 0.22573381662368774,
-      "rewards/true_env_reward_fn/mean": 0.4916413426399231,
-      "rewards/true_env_reward_fn/std": 0.22573381662368774,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 145.0,
+      "completions/mean_length": 76.77083587646484,
+      "completions/mean_terminated_length": 67.51063537597656,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.6089566349983215,
+      "epoch": 8.21951219512195,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07641154527664185,
+      "kl": 0.00011577324767131358,
+      "learning_rate": 9.599251766535343e-08,
+      "loss": -0.047279007732868195,
+      "num_tokens": 8331715.0,
+      "reward": 0.37190374732017517,
+      "reward_std": 0.2730160653591156,
+      "rewards/true_env_reward_fn/mean": 0.37190374732017517,
+      "rewards/true_env_reward_fn/std": 0.2730160653591156,
       "step": 337,
-      "step_time": 9.208789625999998
+      "step_time": 33.08387189299992
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9452,26 +9452,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 50.75,
-      "completions/mean_terminated_length": 50.75,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2942972779273987,
-      "epoch": 2.747967479674797,
+      "completions/max_length": 158.0,
+      "completions/max_terminated_length": 158.0,
+      "completions/mean_length": 60.79166793823242,
+      "completions/mean_terminated_length": 60.79166793823242,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2664433717727661,
+      "epoch": 8.24390243902439,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2804953157901764,
-      "kl": 6.65090201437124e-05,
-      "learning_rate": 1.8198374891377966e-08,
-      "loss": 0.14034238457679749,
-      "num_tokens": 1392442.0,
-      "reward": 0.458748459815979,
-      "reward_std": 0.24402853846549988,
-      "rewards/true_env_reward_fn/mean": 0.458748459815979,
-      "rewards/true_env_reward_fn/std": 0.24402853846549988,
+      "grad_norm": 0.10079872608184814,
+      "kl": 0.00016995913574646693,
+      "learning_rate": 9.349918616258113e-08,
+      "loss": 0.04466569796204567,
+      "num_tokens": 8357801.0,
+      "reward": 0.4597291648387909,
+      "reward_std": 0.2547009587287903,
+      "rewards/true_env_reward_fn/mean": 0.4597291648387909,
+      "rewards/true_env_reward_fn/std": 0.2547009289264679,
       "step": 338,
-      "step_time": 3.383874344001015
+      "step_time": 12.540567380000539
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9480,26 +9480,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 75.625,
-      "completions/mean_terminated_length": 75.625,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "entropy": 1.433261752128601,
-      "epoch": 2.7560975609756095,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11875848472118378,
-      "kl": 2.3840807443775702e-05,
-      "learning_rate": 1.7086790704341402e-08,
-      "loss": -0.015734028071165085,
-      "num_tokens": 1396235.0,
-      "reward": 0.3571140468120575,
-      "reward_std": 0.10167346149682999,
-      "rewards/true_env_reward_fn/mean": 0.3571140468120575,
-      "rewards/true_env_reward_fn/std": 0.10167345404624939,
+      "completions/max_length": 278.0,
+      "completions/max_terminated_length": 278.0,
+      "completions/mean_length": 72.95833587646484,
+      "completions/mean_terminated_length": 72.95833587646484,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.20101597905159,
+      "epoch": 8.268292682926829,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0724605992436409,
+      "kl": 8.693387826497201e-05,
+      "learning_rate": 9.103531961664118e-08,
+      "loss": -0.04201607406139374,
+      "num_tokens": 8391047.0,
+      "reward": 0.272682249546051,
+      "reward_std": 0.22370731830596924,
+      "rewards/true_env_reward_fn/mean": 0.27268221974372864,
+      "rewards/true_env_reward_fn/std": 0.22370733320713043,
       "step": 339,
-      "step_time": 4.3859739109993825
+      "step_time": 21.499952529999973
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9508,26 +9508,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 186.0,
-      "completions/max_terminated_length": 186.0,
-      "completions/mean_length": 97.75,
-      "completions/mean_terminated_length": 97.75,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.5857495069503784,
-      "epoch": 2.7642276422764227,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.698282959405333e-05,
-      "kl": 1.4666956758446759e-05,
-      "learning_rate": 1.600949282243089e-08,
-      "loss": 7.313840342249023e-07,
-      "num_tokens": 1401073.0,
-      "reward": 0.3621603846549988,
-      "reward_std": 0.10388225317001343,
-      "rewards/true_env_reward_fn/mean": 0.3621603846549988,
-      "rewards/true_env_reward_fn/std": 0.10388225317001343,
+      "completions/max_length": 152.0,
+      "completions/max_terminated_length": 152.0,
+      "completions/mean_length": 66.27083587646484,
+      "completions/mean_terminated_length": 66.27083587646484,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.240132600069046,
+      "epoch": 8.292682926829269,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09206120669841766,
+      "kl": 9.196988685289398e-05,
+      "learning_rate": 8.860109661935672e-08,
+      "loss": 0.013995083048939705,
+      "num_tokens": 8412060.0,
+      "reward": 0.5260962843894958,
+      "reward_std": 0.16053001582622528,
+      "rewards/true_env_reward_fn/mean": 0.5260962843894958,
+      "rewards/true_env_reward_fn/std": 0.16053001582622528,
       "step": 340,
-      "step_time": 7.751754675999109
+      "step_time": 11.287615471999743
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9536,26 +9536,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 54.0,
-      "completions/mean_terminated_length": 54.0,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.0097725987434387,
-      "epoch": 2.772357723577236,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011211638775421306,
-      "kl": 1.3342013517103624e-05,
-      "learning_rate": 1.496657770762013e-08,
-      "loss": 6.635225417994661e-07,
-      "num_tokens": 1403921.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/max_length": 187.0,
+      "completions/max_terminated_length": 187.0,
+      "completions/mean_length": 69.85417175292969,
+      "completions/mean_terminated_length": 69.85417175292969,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.2434665858745575,
+      "epoch": 8.317073170731707,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06606078892946243,
+      "kl": 0.00013524454698199406,
+      "learning_rate": 8.619669361385662e-08,
+      "loss": 0.04330434650182724,
+      "num_tokens": 8435189.0,
+      "reward": 0.5729736089706421,
+      "reward_std": 0.28772154450416565,
+      "rewards/true_env_reward_fn/mean": 0.5729735493659973,
+      "rewards/true_env_reward_fn/std": 0.28772154450416565,
       "step": 341,
-      "step_time": 3.4684136910000234
+      "step_time": 14.554103153000142
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9564,26 +9564,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.1732905507087708,
-      "epoch": 2.7804878048780486,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14162735641002655,
-      "kl": 4.145995080762077e-05,
-      "learning_rate": 1.3958138743226688e-08,
-      "loss": 0.007000558078289032,
-      "num_tokens": 1407681.0,
-      "reward": 0.5386021733283997,
-      "reward_std": 0.31964072585105896,
-      "rewards/true_env_reward_fn/mean": 0.5386021733283997,
-      "rewards/true_env_reward_fn/std": 0.31964072585105896,
+      "completions/max_length": 185.0,
+      "completions/max_terminated_length": 185.0,
+      "completions/mean_length": 87.83333587646484,
+      "completions/mean_terminated_length": 87.83333587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.193087100982666,
+      "epoch": 8.341463414634147,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.06404980272054672,
+      "kl": 7.958761125337332e-05,
+      "learning_rate": 8.382228488178638e-08,
+      "loss": -0.02654789574444294,
+      "num_tokens": 8461485.0,
+      "reward": 0.4391756057739258,
+      "reward_std": 0.3396260142326355,
+      "rewards/true_env_reward_fn/mean": 0.4391756057739258,
+      "rewards/true_env_reward_fn/std": 0.3396260142326355,
       "step": 342,
-      "step_time": 3.5975158429992007
+      "step_time": 16.93046250899988
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9592,26 +9592,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 98.0,
-      "completions/max_terminated_length": 98.0,
-      "completions/mean_length": 67.625,
-      "completions/mean_terminated_length": 67.625,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.0845041275024414,
-      "epoch": 2.7886178861788617,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.0979820266366005,
-      "kl": 1.916076598718064e-05,
-      "learning_rate": 1.29842662255506e-08,
-      "loss": -0.049884289503097534,
-      "num_tokens": 1413182.0,
-      "reward": 0.316629558801651,
-      "reward_std": 0.16504530608654022,
-      "rewards/true_env_reward_fn/mean": 0.316629558801651,
-      "rewards/true_env_reward_fn/std": 0.16504532098770142,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 66.0625,
+      "completions/mean_terminated_length": 66.0625,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2198111712932587,
+      "epoch": 8.365853658536585,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06700363755226135,
+      "kl": 3.631071058407542e-05,
+      "learning_rate": 8.14780425306758e-08,
+      "loss": -0.01248769462108612,
+      "num_tokens": 8477128.0,
+      "reward": 0.6880009174346924,
+      "reward_std": 0.16646623611450195,
+      "rewards/true_env_reward_fn/mean": 0.6880009174346924,
+      "rewards/true_env_reward_fn/std": 0.16646623611450195,
       "step": 343,
-      "step_time": 4.6961259659983625
+      "step_time": 8.379146702999606
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9620,26 +9620,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 51.25,
-      "completions/mean_terminated_length": 51.25,
-      "completions/min_length": 23.0,
-      "completions/min_terminated_length": 23.0,
-      "entropy": 1.1820043921470642,
-      "epoch": 2.796747967479675,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1726708859205246,
-      "kl": 2.1745819140051026e-05,
-      "learning_rate": 1.2045047355789107e-08,
-      "loss": 0.048721786588430405,
-      "num_tokens": 1415472.0,
-      "reward": 0.6344382762908936,
-      "reward_std": 0.07378704100847244,
-      "rewards/true_env_reward_fn/mean": 0.6344382762908936,
-      "rewards/true_env_reward_fn/std": 0.07378704845905304,
+      "completions/max_length": 240.0,
+      "completions/max_terminated_length": 240.0,
+      "completions/mean_length": 72.0625,
+      "completions/mean_terminated_length": 72.0625,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "entropy": 1.365002065896988,
+      "epoch": 8.390243902439025,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07387524098157883,
+      "kl": 7.906068458396476e-05,
+      "learning_rate": 7.91641364814628e-08,
+      "loss": -0.019427858293056488,
+      "num_tokens": 8502659.0,
+      "reward": 0.49987101554870605,
+      "reward_std": 0.22534418106079102,
+      "rewards/true_env_reward_fn/mean": 0.49987101554870605,
+      "rewards/true_env_reward_fn/std": 0.22534416615962982,
       "step": 344,
-      "step_time": 3.2691068980002456
+      "step_time": 16.447989433999282
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9648,26 +9648,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 59.625,
-      "completions/mean_terminated_length": 59.625,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.0933732390403748,
-      "epoch": 2.8048780487804876,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21739810705184937,
-      "kl": 2.0197898265905678e-05,
-      "learning_rate": 1.1140566232228677e-08,
-      "loss": 0.0031427331268787384,
-      "num_tokens": 1417385.0,
-      "reward": 0.5309104323387146,
-      "reward_std": 0.18906596302986145,
-      "rewards/true_env_reward_fn/mean": 0.5309104323387146,
-      "rewards/true_env_reward_fn/std": 0.18906594812870026,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 63.833335876464844,
+      "completions/mean_terminated_length": 63.833335876464844,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3033632636070251,
+      "epoch": 8.414634146341463,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07896492630243301,
+      "kl": 0.00015640226229152177,
+      "learning_rate": 7.688073445617798e-08,
+      "loss": 0.032075606286525726,
+      "num_tokens": 8527771.0,
+      "reward": 0.4142627418041229,
+      "reward_std": 0.39965084195137024,
+      "rewards/true_env_reward_fn/mean": 0.4142627418041229,
+      "rewards/true_env_reward_fn/std": 0.3996508717536926,
       "step": 345,
-      "step_time": 3.9515345389991126
+      "step_time": 10.625701701998878
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9676,26 +9676,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 58.875,
-      "completions/mean_terminated_length": 58.875,
-      "completions/min_length": 28.0,
-      "completions/min_terminated_length": 28.0,
-      "entropy": 1.2968629598617554,
-      "epoch": 2.813008130081301,
+      "completions/max_length": 222.0,
+      "completions/max_terminated_length": 222.0,
+      "completions/mean_length": 59.375,
+      "completions/mean_terminated_length": 59.375,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2410016655921936,
+      "epoch": 8.439024390243903,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20110434293746948,
-      "kl": 4.813342457055114e-05,
-      "learning_rate": 1.0270903842714673e-08,
-      "loss": 0.062390223145484924,
-      "num_tokens": 1423496.0,
-      "reward": 0.14946666359901428,
-      "reward_std": 0.24863645434379578,
-      "rewards/true_env_reward_fn/mean": 0.14946666359901428,
-      "rewards/true_env_reward_fn/std": 0.24863646924495697,
+      "grad_norm": 0.10202007740736008,
+      "kl": 0.00028172876773169264,
+      "learning_rate": 7.462800196578661e-08,
+      "loss": 0.030375666916370392,
+      "num_tokens": 8557085.0,
+      "reward": 0.3401162028312683,
+      "reward_std": 0.2810371518135071,
+      "rewards/true_env_reward_fn/mean": 0.3401161730289459,
+      "rewards/true_env_reward_fn/std": 0.2810371518135071,
       "step": 346,
-      "step_time": 4.331638548001138
+      "step_time": 15.773468264999792
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9704,26 +9704,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 57.0,
-      "completions/min_terminated_length": 57.0,
-      "entropy": 1.153487741947174,
-      "epoch": 2.821138211382114,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011695526336552575,
-      "kl": 1.7154279703390785e-05,
-      "learning_rate": 9.436138057399601e-09,
-      "loss": 8.606326673543663e-07,
-      "num_tokens": 1427182.0,
-      "reward": 0.5776733160018921,
-      "reward_std": 0.11168669909238815,
-      "rewards/true_env_reward_fn/mean": 0.5776733160018921,
-      "rewards/true_env_reward_fn/std": 0.11168669164180756,
+      "completions/max_length": 96.0,
+      "completions/max_terminated_length": 96.0,
+      "completions/mean_length": 62.97916793823242,
+      "completions/mean_terminated_length": 62.97916793823242,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.4290962517261505,
+      "epoch": 8.463414634146341,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06221809610724449,
+      "kl": 9.756034887686837e-05,
+      "learning_rate": 7.240610229819195e-08,
+      "loss": -0.009350121021270752,
+      "num_tokens": 8583644.0,
+      "reward": 0.42270374298095703,
+      "reward_std": 0.19876612722873688,
+      "rewards/true_env_reward_fn/mean": 0.42270374298095703,
+      "rewards/true_env_reward_fn/std": 0.19876612722873688,
       "step": 347,
-      "step_time": 3.416529175998221
+      "step_time": 10.18261290999908
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9732,26 +9732,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 61.125,
-      "completions/mean_terminated_length": 61.125,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.3210760354995728,
-      "epoch": 2.8292682926829267,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13752248883247375,
-      "kl": 3.656550507002976e-05,
-      "learning_rate": 8.636343621770948e-09,
-      "loss": -0.029253121465444565,
-      "num_tokens": 1431839.0,
-      "reward": 0.47378063201904297,
-      "reward_std": 0.32937321066856384,
-      "rewards/true_env_reward_fn/mean": 0.47378063201904297,
-      "rewards/true_env_reward_fn/std": 0.32937324047088623,
+      "completions/max_length": 361.0,
+      "completions/max_terminated_length": 361.0,
+      "completions/mean_length": 68.89583587646484,
+      "completions/mean_terminated_length": 68.89583587646484,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2232879400253296,
+      "epoch": 8.487804878048781,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07711587101221085,
+      "kl": 7.56127965360065e-05,
+      "learning_rate": 7.021519650639951e-08,
+      "loss": -0.06937319040298462,
+      "num_tokens": 8604055.0,
+      "reward": 0.5483574867248535,
+      "reward_std": 0.2410479635000229,
+      "rewards/true_env_reward_fn/mean": 0.5483574867248535,
+      "rewards/true_env_reward_fn/std": 0.2410479635000229,
       "step": 348,
-      "step_time": 3.957806388998506
+      "step_time": 20.953229555999314
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9760,26 +9760,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 129.0,
-      "completions/max_terminated_length": 129.0,
-      "completions/mean_length": 83.75,
-      "completions/mean_terminated_length": 83.75,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.3464251160621643,
-      "epoch": 2.83739837398374,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11163587868213654,
-      "kl": 1.8057865418086294e-05,
-      "learning_rate": 7.87159214995774e-09,
-      "loss": 0.01491672545671463,
-      "num_tokens": 1434785.0,
-      "reward": 0.6471105217933655,
-      "reward_std": 0.1651550978422165,
-      "rewards/true_env_reward_fn/mean": 0.6471105217933655,
-      "rewards/true_env_reward_fn/std": 0.1651550978422165,
+      "completions/max_length": 166.0,
+      "completions/max_terminated_length": 166.0,
+      "completions/mean_length": 74.375,
+      "completions/mean_terminated_length": 74.375,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 1.2409018576145172,
+      "epoch": 8.512195121951219,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08634211868047714,
+      "kl": 0.000148440574776032,
+      "learning_rate": 6.805544339684293e-08,
+      "loss": 0.05703897029161453,
+      "num_tokens": 8636849.0,
+      "reward": 0.2619777023792267,
+      "reward_std": 0.3443836271762848,
+      "rewards/true_env_reward_fn/mean": 0.2619777023792267,
+      "rewards/true_env_reward_fn/std": 0.3443836271762848,
       "step": 349,
-      "step_time": 5.513293031999638
+      "step_time": 13.73512468699937
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -9788,32 +9788,32 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 57.0,
-      "completions/mean_terminated_length": 57.0,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.1192307472229004,
-      "epoch": 2.845528455284553,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20383992791175842,
-      "kl": 1.563574414831237e-05,
-      "learning_rate": 7.141952118318961e-09,
-      "loss": 0.113973468542099,
-      "num_tokens": 1437333.0,
-      "reward": 0.5760529637336731,
-      "reward_std": 0.11078595370054245,
-      "rewards/true_env_reward_fn/mean": 0.5760529637336731,
-      "rewards/true_env_reward_fn/std": 0.11078596115112305,
+      "completions/max_length": 225.0,
+      "completions/max_terminated_length": 225.0,
+      "completions/mean_length": 64.85417175292969,
+      "completions/mean_terminated_length": 64.85417175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2963348627090454,
+      "epoch": 8.536585365853659,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09569360315799713,
+      "kl": 0.0001818603413994424,
+      "learning_rate": 6.592699951787362e-08,
+      "loss": 0.0750311091542244,
+      "num_tokens": 8668938.0,
+      "reward": 0.2865946292877197,
+      "reward_std": 0.21787859499454498,
+      "rewards/true_env_reward_fn/mean": 0.2865946292877197,
+      "rewards/true_env_reward_fn/std": 0.21787859499454498,
       "step": 350,
-      "step_time": 3.624226386000373
+      "step_time": 16.08931997000036
     }
   ],
   "logging_steps": 1,
-  "max_steps": 369,
-  "num_input_tokens_seen": 1437333,
-  "num_train_epochs": 3,
+  "max_steps": 410,
+  "num_input_tokens_seen": 8668938,
+  "num_train_epochs": 10,
   "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -9828,7 +9828,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 12,
   "trial_name": null,
   "trial_params": null
 }