diff --git "a/checkpoint-150/trainer_state.json" "b/checkpoint-150/trainer_state.json"
--- "a/checkpoint-150/trainer_state.json"
+++ "b/checkpoint-150/trainer_state.json"
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.2195121951219512,
+  "epoch": 3.658536585365854,
   "eval_steps": 500,
   "global_step": 150,
   "is_hyper_param_search": false,
@@ -16,26 +16,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 56.25,
-      "completions/mean_terminated_length": 56.25,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1208415031433105,
-      "epoch": 0.008130081300813009,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2371738702058792,
-      "kl": 1.1247546808590414e-05,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 60.10416793823242,
+      "completions/mean_terminated_length": 60.10416793823242,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2584454119205475,
+      "epoch": 0.024390243902439025,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09302648901939392,
+      "kl": 1.2248776783962967e-05,
       "learning_rate": 0.0,
-      "loss": 0.019987963140010834,
-      "num_tokens": 2250.0,
-      "reward": 0.7777429223060608,
-      "reward_std": 0.14680756628513336,
-      "rewards/true_env_reward_fn/mean": 0.7777429223060608,
-      "rewards/true_env_reward_fn/std": 0.14680756628513336,
+      "loss": -0.0423424206674099,
+      "num_tokens": 23029.0,
+      "reward": 0.5082165002822876,
+      "reward_std": 0.27811428904533386,
+      "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+      "rewards/true_env_reward_fn/std": 0.27811428904533386,
       "step": 1,
-      "step_time": 3.622400252999796
+      "step_time": 11.815711200999885
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 126.0,
-      "completions/max_terminated_length": 126.0,
-      "completions/mean_length": 72.875,
-      "completions/mean_terminated_length": 72.875,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 0.9768376648426056,
-      "epoch": 0.016260162601626018,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10255444794893265,
-      "kl": 9.209406016452704e-06,
-      "learning_rate": 2.1621621621621623e-08,
-      "loss": -0.16146813333034515,
-      "num_tokens": 8517.0,
-      "reward": 0.5471514463424683,
-      "reward_std": 0.19726651906967163,
-      "rewards/true_env_reward_fn/mean": 0.5471514463424683,
-      "rewards/true_env_reward_fn/std": 0.19726651906967163,
+      "completions/max_length": 161.0,
+      "completions/max_terminated_length": 161.0,
+      "completions/mean_length": 55.875,
+      "completions/mean_terminated_length": 55.875,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "entropy": 1.3789870142936707,
+      "epoch": 0.04878048780487805,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11938872188329697,
+      "kl": 1.2672078355535632e-05,
+      "learning_rate": 2.4390243902439023e-08,
+      "loss": -0.11833255738019943,
+      "num_tokens": 57015.0,
+      "reward": 0.1327376663684845,
+      "reward_std": 0.241567462682724,
+      "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+      "rewards/true_env_reward_fn/std": 0.241567462682724,
       "step": 2,
-      "step_time": 5.979386726001394
+      "step_time": 13.493524850000085
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 67.125,
-      "completions/mean_terminated_length": 67.125,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.324017882347107,
-      "epoch": 0.024390243902439025,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13558730483055115,
-      "kl": 1.2776082257914823e-05,
-      "learning_rate": 4.3243243243243246e-08,
-      "loss": 0.008048340678215027,
-      "num_tokens": 11858.0,
-      "reward": 0.5399026870727539,
-      "reward_std": 0.04722921922802925,
-      "rewards/true_env_reward_fn/mean": 0.5399026870727539,
-      "rewards/true_env_reward_fn/std": 0.047229230403900146,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 63.79166793823242,
+      "completions/mean_terminated_length": 63.79166793823242,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "entropy": 1.315225213766098,
+      "epoch": 0.07317073170731707,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08519645780324936,
+      "kl": 1.2407871281538974e-05,
+      "learning_rate": 4.878048780487805e-08,
+      "loss": -0.03654177859425545,
+      "num_tokens": 86989.0,
+      "reward": 0.3152047097682953,
+      "reward_std": 0.3069385886192322,
+      "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+      "rewards/true_env_reward_fn/std": 0.30693864822387695,
       "step": 3,
-      "step_time": 3.6805073480009014
+      "step_time": 11.449303891999875
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 53.625,
-      "completions/mean_terminated_length": 53.625,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.0729783773422241,
-      "epoch": 0.032520325203252036,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23012493550777435,
-      "kl": 1.0804946214193478e-05,
-      "learning_rate": 6.486486486486487e-08,
-      "loss": 0.13091428577899933,
-      "num_tokens": 15379.0,
-      "reward": 0.4351762533187866,
-      "reward_std": 0.2320314645767212,
-      "rewards/true_env_reward_fn/mean": 0.4351762533187866,
-      "rewards/true_env_reward_fn/std": 0.2320314645767212,
+      "completions/max_length": 226.0,
+      "completions/max_terminated_length": 226.0,
+      "completions/mean_length": 77.20833587646484,
+      "completions/mean_terminated_length": 77.20833587646484,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.338063895702362,
+      "epoch": 0.0975609756097561,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08080132305622101,
+      "kl": 1.239982589140709e-05,
+      "learning_rate": 7.317073170731706e-08,
+      "loss": 0.053779490292072296,
+      "num_tokens": 112007.0,
+      "reward": 0.4893929362297058,
+      "reward_std": 0.28476035594940186,
+      "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+      "rewards/true_env_reward_fn/std": 0.28476035594940186,
       "step": 4,
-      "step_time": 3.421140036001816
+      "step_time": 18.835909622000145
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 57.25,
-      "completions/mean_terminated_length": 57.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.3000869154930115,
-      "epoch": 0.04065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22867721319198608,
-      "kl": 1.2170262834843015e-05,
-      "learning_rate": 8.648648648648649e-08,
-      "loss": 0.08851668983697891,
-      "num_tokens": 19401.0,
-      "reward": 0.4031979441642761,
-      "reward_std": 0.32033228874206543,
-      "rewards/true_env_reward_fn/mean": 0.4031979441642761,
-      "rewards/true_env_reward_fn/std": 0.32033228874206543,
+      "completions/max_length": 212.0,
+      "completions/max_terminated_length": 212.0,
+      "completions/mean_length": 67.41667175292969,
+      "completions/mean_terminated_length": 67.41667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3855182826519012,
+      "epoch": 0.12195121951219512,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08783729374408722,
+      "kl": 1.1660237760224845e-05,
+      "learning_rate": 9.75609756097561e-08,
+      "loss": -0.026884621009230614,
+      "num_tokens": 135883.0,
+      "reward": 0.48575252294540405,
+      "reward_std": 0.335994690656662,
+      "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+      "rewards/true_env_reward_fn/std": 0.335994690656662,
       "step": 5,
-      "step_time": 3.977350764000221
+      "step_time": 14.435845696000001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 58.5,
-      "completions/mean_terminated_length": 58.5,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.1719728112220764,
-      "epoch": 0.04878048780487805,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16202858090400696,
-      "kl": 1.228428209287813e-05,
-      "learning_rate": 1.0810810810810811e-07,
-      "loss": 0.1666201949119568,
-      "num_tokens": 21253.0,
-      "reward": 0.5827490091323853,
-      "reward_std": 0.27126544713974,
-      "rewards/true_env_reward_fn/mean": 0.5827490091323853,
-      "rewards/true_env_reward_fn/std": 0.27126544713974,
+      "completions/max_length": 164.0,
+      "completions/max_terminated_length": 164.0,
+      "completions/mean_length": 71.29167175292969,
+      "completions/mean_terminated_length": 71.29167175292969,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2962585091590881,
+      "epoch": 0.14634146341463414,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08510823547840118,
+      "kl": 1.241418908648484e-05,
+      "learning_rate": 1.219512195121951e-07,
+      "loss": -0.05353507027029991,
+      "num_tokens": 157537.0,
+      "reward": 0.47622889280319214,
+      "reward_std": 0.3605790138244629,
+      "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+      "rewards/true_env_reward_fn/std": 0.3605790138244629,
       "step": 6,
-      "step_time": 4.179320960000041
+      "step_time": 13.232063896999989
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.3112086653709412,
-      "epoch": 0.056910569105691054,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20938768982887268,
-      "kl": 1.306734156969469e-05,
-      "learning_rate": 1.2972972972972974e-07,
-      "loss": 0.04748187214136124,
-      "num_tokens": 25726.0,
-      "reward": 0.2716812491416931,
-      "reward_std": 0.29254475235939026,
-      "rewards/true_env_reward_fn/mean": 0.2716812491416931,
-      "rewards/true_env_reward_fn/std": 0.29254478216171265,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 69.45833587646484,
+      "completions/mean_terminated_length": 69.45833587646484,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.273663192987442,
+      "epoch": 0.17073170731707318,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0775279700756073,
+      "kl": 1.2900356978207128e-05,
+      "learning_rate": 1.4634146341463413e-07,
+      "loss": -0.010494321584701538,
+      "num_tokens": 179167.0,
+      "reward": 0.5062826871871948,
+      "reward_std": 0.18032674491405487,
+      "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+      "rewards/true_env_reward_fn/std": 0.18032673001289368,
       "step": 7,
-      "step_time": 3.438178512999002
+      "step_time": 9.810652986000036
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 78.5,
-      "completions/mean_terminated_length": 78.5,
-      "completions/min_length": 65.0,
-      "completions/min_terminated_length": 65.0,
-      "entropy": 1.2046615481376648,
-      "epoch": 0.06504065040650407,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.857898890506476e-05,
-      "kl": 1.3628536180476658e-05,
-      "learning_rate": 1.5135135135135135e-07,
-      "loss": 6.865971045044716e-07,
-      "num_tokens": 30126.0,
-      "reward": 0.49959999322891235,
-      "reward_std": 0.02822280302643776,
-      "rewards/true_env_reward_fn/mean": 0.49959999322891235,
-      "rewards/true_env_reward_fn/std": 0.02822280302643776,
+      "completions/max_length": 287.0,
+      "completions/max_terminated_length": 287.0,
+      "completions/mean_length": 65.54167175292969,
+      "completions/mean_terminated_length": 65.54167175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.255563884973526,
+      "epoch": 0.1951219512195122,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07464194297790527,
+      "kl": 1.1561841347429436e-05,
+      "learning_rate": 1.7073170731707317e-07,
+      "loss": 0.0830899029970169,
+      "num_tokens": 201865.0,
+      "reward": 0.38212963938713074,
+      "reward_std": 0.29894331097602844,
+      "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+      "rewards/true_env_reward_fn/std": 0.29894331097602844,
       "step": 8,
-      "step_time": 4.324984626000514
+      "step_time": 19.874756868999953
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 62.75,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.2111859917640686,
-      "epoch": 0.07317073170731707,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11552055925130844,
-      "kl": 1.0166647598452982e-05,
-      "learning_rate": 1.7297297297297298e-07,
-      "loss": -0.03883127495646477,
-      "num_tokens": 33332.0,
-      "reward": 0.571246862411499,
-      "reward_std": 0.2893567681312561,
-      "rewards/true_env_reward_fn/mean": 0.571246862411499,
-      "rewards/true_env_reward_fn/std": 0.2893567681312561,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 68.33333587646484,
+      "completions/mean_terminated_length": 68.33333587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2706169188022614,
+      "epoch": 0.21951219512195122,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.049192048609256744,
+      "kl": 1.157601468548819e-05,
+      "learning_rate": 1.951219512195122e-07,
+      "loss": 0.010864660143852234,
+      "num_tokens": 219953.0,
+      "reward": 0.6740004420280457,
+      "reward_std": 0.18809831142425537,
+      "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+      "rewards/true_env_reward_fn/std": 0.18809829652309418,
       "step": 9,
-      "step_time": 3.651253555999574
+      "step_time": 9.458149736999985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 58.125,
-      "completions/mean_terminated_length": 58.125,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.195803463459015,
-      "epoch": 0.08130081300813008,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14317302405834198,
-      "kl": 1.476421539337025e-05,
-      "learning_rate": 1.945945945945946e-07,
-      "loss": -0.034387920051813126,
-      "num_tokens": 36381.0,
-      "reward": 0.5593140125274658,
-      "reward_std": 0.42223072052001953,
-      "rewards/true_env_reward_fn/mean": 0.5593140125274658,
-      "rewards/true_env_reward_fn/std": 0.42223072052001953,
+      "completions/max_length": 148.0,
+      "completions/max_terminated_length": 148.0,
+      "completions/mean_length": 59.833335876464844,
+      "completions/mean_terminated_length": 59.833335876464844,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.1927059888839722,
+      "epoch": 0.24390243902439024,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.0561092346906662,
+      "kl": 1.0622535000948119e-05,
+      "learning_rate": 2.195121951219512e-07,
+      "loss": -0.02407176047563553,
+      "num_tokens": 244913.0,
+      "reward": 0.5113257169723511,
+      "reward_std": 0.32156965136528015,
+      "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+      "rewards/true_env_reward_fn/std": 0.32156962156295776,
       "step": 10,
-      "step_time": 3.8103441190014564
+      "step_time": 14.219840567000006
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 61.625,
-      "completions/mean_terminated_length": 61.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.389159917831421,
-      "epoch": 0.08943089430894309,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 65.47917175292969,
+      "completions/mean_terminated_length": 65.47917175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2782267928123474,
+      "epoch": 0.2682926829268293,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11816789954900742,
-      "kl": 1.0807365470100194e-05,
-      "learning_rate": 2.1621621621621622e-07,
-      "loss": 0.04055345058441162,
-      "num_tokens": 41830.0,
-      "reward": 0.12224999815225601,
-      "reward_std": 0.27913153171539307,
-      "rewards/true_env_reward_fn/mean": 0.12224999815225601,
-      "rewards/true_env_reward_fn/std": 0.27913153171539307,
+      "grad_norm": 0.05816411226987839,
+      "kl": 1.2071807759639341e-05,
+      "learning_rate": 2.439024390243902e-07,
+      "loss": 0.007693461142480373,
+      "num_tokens": 269080.0,
+      "reward": 0.37106746435165405,
+      "reward_std": 0.26608046889305115,
+      "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+      "rewards/true_env_reward_fn/std": 0.26608046889305115,
       "step": 11,
-      "step_time": 4.204996996000773
+      "step_time": 9.271131832999913
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.3866143822669983,
-      "epoch": 0.0975609756097561,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19537723064422607,
-      "kl": 1.5072046608111123e-05,
-      "learning_rate": 2.3783783783783783e-07,
-      "loss": -0.07735465466976166,
-      "num_tokens": 47047.0,
-      "reward": 0.3571999967098236,
-      "reward_std": 0.18295250833034515,
-      "rewards/true_env_reward_fn/mean": 0.3571999967098236,
-      "rewards/true_env_reward_fn/std": 0.18295250833034515,
+      "completions/max_length": 143.0,
+      "completions/max_terminated_length": 143.0,
+      "completions/mean_length": 67.9375,
+      "completions/mean_terminated_length": 67.9375,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3190773129463196,
+      "epoch": 0.2926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09140665084123611,
+      "kl": 1.2069708191120299e-05,
+      "learning_rate": 2.682926829268293e-07,
+      "loss": 0.07185906916856766,
+      "num_tokens": 291317.0,
+      "reward": 0.4376159906387329,
+      "reward_std": 0.27247554063796997,
+      "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+      "rewards/true_env_reward_fn/std": 0.27247554063796997,
       "step": 12,
-      "step_time": 4.775358541999594
+      "step_time": 12.184364300000084
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 55.75,
-      "completions/mean_terminated_length": 55.75,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.1633875966072083,
-      "epoch": 0.10569105691056911,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14852823317050934,
-      "kl": 1.4038786503078882e-05,
-      "learning_rate": 2.594594594594595e-07,
-      "loss": -0.04705440253019333,
-      "num_tokens": 51521.0,
-      "reward": 0.44465911388397217,
-      "reward_std": 0.15160730481147766,
-      "rewards/true_env_reward_fn/mean": 0.44465911388397217,
-      "rewards/true_env_reward_fn/std": 0.15160730481147766,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 66.54167175292969,
+      "completions/mean_terminated_length": 66.54167175292969,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.3555113077163696,
+      "epoch": 0.3170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08824986964464188,
+      "kl": 1.2127976788178785e-05,
+      "learning_rate": 2.9268292682926825e-07,
+      "loss": -0.0217185840010643,
+      "num_tokens": 313623.0,
+      "reward": 0.5092746615409851,
+      "reward_std": 0.3137436807155609,
+      "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+      "rewards/true_env_reward_fn/std": 0.3137436509132385,
       "step": 13,
-      "step_time": 4.072596639998665
+      "step_time": 10.720424850000086
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 60.375,
-      "completions/mean_terminated_length": 60.375,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.2392634153366089,
-      "epoch": 0.11382113821138211,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23616985976696014,
-      "kl": 1.3279905488161603e-05,
-      "learning_rate": 2.810810810810811e-07,
-      "loss": -0.021731968969106674,
-      "num_tokens": 55556.0,
-      "reward": 0.4130214750766754,
-      "reward_std": 0.43705809116363525,
-      "rewards/true_env_reward_fn/mean": 0.4130214750766754,
-      "rewards/true_env_reward_fn/std": 0.43705806136131287,
+      "completions/max_length": 188.0,
+      "completions/max_terminated_length": 188.0,
+      "completions/mean_length": 69.3125,
+      "completions/mean_terminated_length": 69.3125,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "entropy": 1.3283279240131378,
+      "epoch": 0.34146341463414637,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05055573210120201,
+      "kl": 1.3128182672517141e-05,
+      "learning_rate": 3.170731707317073e-07,
+      "loss": -0.024722743779420853,
+      "num_tokens": 339118.0,
+      "reward": 0.45545920729637146,
+      "reward_std": 0.18457132577896118,
+      "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+      "rewards/true_env_reward_fn/std": 0.18457134068012238,
       "step": 14,
-      "step_time": 3.7906999759998143
+      "step_time": 14.965493325000011
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 61.5,
-      "completions/mean_terminated_length": 61.5,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.1012902855873108,
-      "epoch": 0.12195121951219512,
+      "completions/max_length": 181.0,
+      "completions/max_terminated_length": 181.0,
+      "completions/mean_length": 66.45833587646484,
+      "completions/mean_terminated_length": 66.45833587646484,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.2629931271076202,
+      "epoch": 0.36585365853658536,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11504171043634415,
-      "kl": 1.1161580914631486e-05,
-      "learning_rate": 3.027027027027027e-07,
-      "loss": -0.03352496027946472,
-      "num_tokens": 58644.0,
-      "reward": 0.54444819688797,
-      "reward_std": 0.2691938281059265,
-      "rewards/true_env_reward_fn/mean": 0.54444819688797,
-      "rewards/true_env_reward_fn/std": 0.2691938281059265,
+      "grad_norm": 0.06967486441135406,
+      "kl": 1.1465989928183262e-05,
+      "learning_rate": 3.4146341463414634e-07,
+      "loss": 0.046319857239723206,
+      "num_tokens": 366364.0,
+      "reward": 0.4448578357696533,
+      "reward_std": 0.24966756999492645,
+      "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+      "rewards/true_env_reward_fn/std": 0.24966755509376526,
       "step": 15,
-      "step_time": 3.427628186998845
+      "step_time": 13.628413805999912
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 110.0,
-      "completions/max_terminated_length": 110.0,
-      "completions/mean_length": 66.125,
-      "completions/mean_terminated_length": 66.125,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "entropy": 1.1984660625457764,
-      "epoch": 0.13008130081300814,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12652896344661713,
-      "kl": 1.292689512411016e-05,
-      "learning_rate": 3.243243243243243e-07,
-      "loss": -0.08787620067596436,
-      "num_tokens": 62361.0,
-      "reward": 0.46189582347869873,
-      "reward_std": 0.23188425600528717,
-      "rewards/true_env_reward_fn/mean": 0.46189582347869873,
-      "rewards/true_env_reward_fn/std": 0.23188428580760956,
+      "completions/max_length": 194.0,
+      "completions/max_terminated_length": 194.0,
+      "completions/mean_length": 69.04167175292969,
+      "completions/mean_terminated_length": 69.04167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2793545722961426,
+      "epoch": 0.3902439024390244,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.04725664108991623,
+      "kl": 1.1130929124192335e-05,
+      "learning_rate": 3.6585365853658536e-07,
+      "loss": 0.006799306720495224,
+      "num_tokens": 392926.0,
+      "reward": 0.414639949798584,
+      "reward_std": 0.2748004198074341,
+      "rewards/true_env_reward_fn/mean": 0.414639949798584,
+      "rewards/true_env_reward_fn/std": 0.2748004198074341,
       "step": 16,
-      "step_time": 4.776189491001787
+      "step_time": 14.229579036999894
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 58.25,
-      "completions/mean_terminated_length": 58.25,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1009634137153625,
-      "epoch": 0.13821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1329507976770401,
-      "kl": 1.1219160569453379e-05,
-      "learning_rate": 3.4594594594594597e-07,
-      "loss": 0.07219867408275604,
-      "num_tokens": 65899.0,
-      "reward": 0.65608811378479,
-      "reward_std": 0.2155800759792328,
-      "rewards/true_env_reward_fn/mean": 0.65608811378479,
-      "rewards/true_env_reward_fn/std": 0.21558009088039398,
+      "completions/max_length": 195.0,
+      "completions/max_terminated_length": 195.0,
+      "completions/mean_length": 76.4375,
+      "completions/mean_terminated_length": 76.4375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3106227219104767,
+      "epoch": 0.4146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06872504949569702,
+      "kl": 1.2065312830600305e-05,
+      "learning_rate": 3.902439024390244e-07,
+      "loss": 0.036527130752801895,
+      "num_tokens": 419219.0,
+      "reward": 0.49165210127830505,
+      "reward_std": 0.267509400844574,
+      "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+      "rewards/true_env_reward_fn/std": 0.267509400844574,
       "step": 17,
-      "step_time": 3.525365152998347
+      "step_time": 17.023353198999985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 51.5,
-      "completions/mean_terminated_length": 51.5,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.1671696901321411,
-      "epoch": 0.14634146341463414,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 71.72917175292969,
+      "completions/mean_terminated_length": 71.72917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3780030608177185,
+      "epoch": 0.43902439024390244,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15754961967468262,
-      "kl": 1.3107276572554838e-05,
-      "learning_rate": 3.6756756756756757e-07,
-      "loss": -0.016363894566893578,
-      "num_tokens": 70155.0,
-      "reward": 0.3013000190258026,
-      "reward_std": 0.2883487343788147,
-      "rewards/true_env_reward_fn/mean": 0.3013000190258026,
-      "rewards/true_env_reward_fn/std": 0.2883487641811371,
+      "grad_norm": 0.05453665927052498,
+      "kl": 1.2325858278927626e-05,
+      "learning_rate": 4.146341463414634e-07,
+      "loss": 0.01989848166704178,
+      "num_tokens": 442822.0,
+      "reward": 0.5288735032081604,
+      "reward_std": 0.2950553297996521,
+      "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+      "rewards/true_env_reward_fn/std": 0.2950552701950073,
       "step": 18,
-      "step_time": 4.143123763000403
+      "step_time": 11.965533113999868
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 111.0,
-      "completions/max_terminated_length": 111.0,
-      "completions/mean_length": 70.25,
-      "completions/mean_terminated_length": 70.25,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.266749083995819,
-      "epoch": 0.15447154471544716,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11665906757116318,
-      "kl": 1.2845626315538539e-05,
-      "learning_rate": 3.891891891891892e-07,
-      "loss": -0.11013027280569077,
-      "num_tokens": 73389.0,
-      "reward": 0.6058553457260132,
-      "reward_std": 0.11022671312093735,
-      "rewards/true_env_reward_fn/mean": 0.6058553457260132,
-      "rewards/true_env_reward_fn/std": 0.11022673547267914,
+      "completions/max_length": 106.0,
+      "completions/max_terminated_length": 106.0,
+      "completions/mean_length": 65.4375,
+      "completions/mean_terminated_length": 65.4375,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 1.3424750864505768,
+      "epoch": 0.4634146341463415,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09814280271530151,
+      "kl": 1.2686515219684225e-05,
+      "learning_rate": 4.390243902439024e-07,
+      "loss": 0.06940581649541855,
+      "num_tokens": 467275.0,
+      "reward": 0.5175753831863403,
+      "reward_std": 0.2811976969242096,
+      "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+      "rewards/true_env_reward_fn/std": 0.2811976969242096,
       "step": 19,
-      "step_time": 4.701202698999623
+      "step_time": 10.33812468799988
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 55.125,
-      "completions/mean_terminated_length": 55.125,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.1111925840377808,
-      "epoch": 0.16260162601626016,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1429353505373001,
-      "kl": 8.694359621586045e-06,
-      "learning_rate": 4.108108108108108e-07,
-      "loss": -0.05066477507352829,
-      "num_tokens": 77594.0,
-      "reward": 0.4271581172943115,
-      "reward_std": 0.050101421773433685,
-      "rewards/true_env_reward_fn/mean": 0.4271581172943115,
-      "rewards/true_env_reward_fn/std": 0.05010143294930458,
+      "completions/max_length": 244.0,
+      "completions/max_terminated_length": 244.0,
+      "completions/mean_length": 65.10417175292969,
+      "completions/mean_terminated_length": 65.10417175292969,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.1681120097637177,
+      "epoch": 0.4878048780487805,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09474422037601471,
+      "kl": 1.2183225862827385e-05,
+      "learning_rate": 4.634146341463415e-07,
+      "loss": 0.05423373728990555,
+      "num_tokens": 494320.0,
+      "reward": 0.48628994822502136,
+      "reward_std": 0.25381213426589966,
+      "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+      "rewards/true_env_reward_fn/std": 0.25381216406822205,
       "step": 20,
-      "step_time": 3.220270914998764
+      "step_time": 17.317542748000164
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.0,
-      "completions/max_terminated_length": 68.0,
-      "completions/mean_length": 46.0,
-      "completions/mean_terminated_length": 46.0,
-      "completions/min_length": 19.0,
-      "completions/min_terminated_length": 19.0,
-      "entropy": 1.4938308596611023,
-      "epoch": 0.17073170731707318,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16324248909950256,
-      "kl": 1.1220067335671047e-05,
-      "learning_rate": 4.3243243243243244e-07,
-      "loss": 0.023503631353378296,
-      "num_tokens": 83298.0,
-      "reward": 0.1186770498752594,
-      "reward_std": 0.16449356079101562,
-      "rewards/true_env_reward_fn/mean": 0.1186770498752594,
-      "rewards/true_env_reward_fn/std": 0.16449356079101562,
+      "completions/max_length": 157.0,
+      "completions/max_terminated_length": 157.0,
+      "completions/mean_length": 62.395835876464844,
+      "completions/mean_terminated_length": 62.395835876464844,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.2504475116729736,
+      "epoch": 0.5121951219512195,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0819205492734909,
+      "kl": 1.0698822279664455e-05,
+      "learning_rate": 4.878048780487804e-07,
+      "loss": 0.05607657879590988,
+      "num_tokens": 518323.0,
+      "reward": 0.4693639278411865,
+      "reward_std": 0.32881346344947815,
+      "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+      "rewards/true_env_reward_fn/std": 0.32881346344947815,
       "step": 21,
-      "step_time": 3.451675898999383
+      "step_time": 12.20283881399996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 122.0,
-      "completions/max_terminated_length": 122.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2311039566993713,
-      "epoch": 0.17886178861788618,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23201963305473328,
-      "kl": 1.2657743809540989e-05,
-      "learning_rate": 4.54054054054054e-07,
-      "loss": 0.20273712277412415,
-      "num_tokens": 87825.0,
-      "reward": 0.3444172441959381,
-      "reward_std": 0.4508652687072754,
-      "rewards/true_env_reward_fn/mean": 0.3444172441959381,
-      "rewards/true_env_reward_fn/std": 0.450865238904953,
+      "completions/max_length": 144.0,
+      "completions/max_terminated_length": 144.0,
+      "completions/mean_length": 68.91667175292969,
+      "completions/mean_terminated_length": 68.91667175292969,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2199381291866302,
+      "epoch": 0.5365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06889473646879196,
+      "kl": 1.1745656820494332e-05,
+      "learning_rate": 5.121951219512195e-07,
+      "loss": -0.017973195761442184,
+      "num_tokens": 543591.0,
+      "reward": 0.49388420581817627,
+      "reward_std": 0.2952423393726349,
+      "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+      "rewards/true_env_reward_fn/std": 0.2952423095703125,
       "step": 22,
-      "step_time": 5.440214132999245
+      "step_time": 11.211206898000114
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 65.625,
+      "completions/mean_terminated_length": 65.625,
       "completions/min_length": 41.0,
       "completions/min_terminated_length": 41.0,
-      "entropy": 1.3744811415672302,
-      "epoch": 0.18699186991869918,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12073361128568649,
-      "kl": 1.356211032543797e-05,
-      "learning_rate": 4.7567567567567566e-07,
-      "loss": -0.06243758648633957,
-      "num_tokens": 92940.0,
-      "reward": 0.28657954931259155,
-      "reward_std": 0.19488918781280518,
-      "rewards/true_env_reward_fn/mean": 0.28657954931259155,
-      "rewards/true_env_reward_fn/std": 0.19488917291164398,
+      "entropy": 1.2588726878166199,
+      "epoch": 0.5609756097560976,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08144447952508926,
+      "kl": 1.2306870758038713e-05,
+      "learning_rate": 5.365853658536586e-07,
+      "loss": 0.02826106920838356,
+      "num_tokens": 567973.0,
+      "reward": 0.48142755031585693,
+      "reward_std": 0.26756224036216736,
+      "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+      "rewards/true_env_reward_fn/std": 0.26756221055984497,
       "step": 23,
-      "step_time": 4.085832714999924
+      "step_time": 10.428452587999914
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 100.0,
-      "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 68.875,
-      "completions/mean_terminated_length": 68.875,
-      "completions/min_length": 23.0,
-      "completions/min_terminated_length": 23.0,
-      "entropy": 1.3229535818099976,
-      "epoch": 0.1951219512195122,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.64079047460109e-05,
-      "kl": 1.1639681815722724e-05,
-      "learning_rate": 4.972972972972973e-07,
-      "loss": 5.819025545861223e-07,
-      "num_tokens": 94923.0,
-      "reward": 0.7253252267837524,
-      "reward_std": 0.046159788966178894,
-      "rewards/true_env_reward_fn/mean": 0.7253252267837524,
-      "rewards/true_env_reward_fn/std": 0.046159788966178894,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 59.5625,
+      "completions/mean_terminated_length": 59.5625,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.384379804134369,
+      "epoch": 0.5853658536585366,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.11170398443937302,
+      "kl": 1.2296073691686615e-05,
+      "learning_rate": 5.609756097560975e-07,
+      "loss": 0.07271970808506012,
+      "num_tokens": 590248.0,
+      "reward": 0.38166365027427673,
+      "reward_std": 0.34809473156929016,
+      "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+      "rewards/true_env_reward_fn/std": 0.3480947017669678,
       "step": 24,
-      "step_time": 4.218084741001803
+      "step_time": 11.223491792000118
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 113.0,
-      "completions/max_terminated_length": 113.0,
-      "completions/mean_length": 76.375,
-      "completions/mean_terminated_length": 76.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.3325599431991577,
-      "epoch": 0.2032520325203252,
+      "completions/max_length": 123.0,
+      "completions/max_terminated_length": 123.0,
+      "completions/mean_length": 63.35416793823242,
+      "completions/mean_terminated_length": 63.35416793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.3013385236263275,
+      "epoch": 0.6097560975609756,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17998270690441132,
-      "kl": 1.4024041774973739e-05,
-      "learning_rate": 5.18918918918919e-07,
-      "loss": 0.13167564570903778,
-      "num_tokens": 99178.0,
-      "reward": 0.44252532720565796,
-      "reward_std": 0.1883804053068161,
-      "rewards/true_env_reward_fn/mean": 0.44252532720565796,
-      "rewards/true_env_reward_fn/std": 0.1883804053068161,
+      "grad_norm": 0.10069931298494339,
+      "kl": 1.2947949016961502e-05,
+      "learning_rate": 5.853658536585365e-07,
+      "loss": 0.033605337142944336,
+      "num_tokens": 615345.0,
+      "reward": 0.5046355724334717,
+      "reward_std": 0.2754679322242737,
+      "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+      "rewards/true_env_reward_fn/std": 0.2754679322242737,
       "step": 25,
-      "step_time": 4.84537445100068
+      "step_time": 10.92509102200006
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 64.375,
-      "completions/mean_terminated_length": 64.375,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.2788519263267517,
-      "epoch": 0.21138211382113822,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1275048851966858,
-      "kl": 1.1262640327913687e-05,
-      "learning_rate": 5.405405405405405e-07,
-      "loss": -0.010535649955272675,
-      "num_tokens": 102353.0,
-      "reward": 0.3852383494377136,
-      "reward_std": 0.2447713315486908,
-      "rewards/true_env_reward_fn/mean": 0.3852383494377136,
-      "rewards/true_env_reward_fn/std": 0.244771346449852,
+      "completions/max_length": 101.0,
+      "completions/max_terminated_length": 101.0,
+      "completions/mean_length": 61.41666793823242,
+      "completions/mean_terminated_length": 61.41666793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2652399837970734,
+      "epoch": 0.6341463414634146,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07595694065093994,
+      "kl": 1.151612354988174e-05,
+      "learning_rate": 6.097560975609756e-07,
+      "loss": 0.04607678949832916,
+      "num_tokens": 644749.0,
+      "reward": 0.3311978578567505,
+      "reward_std": 0.21527718007564545,
+      "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+      "rewards/true_env_reward_fn/std": 0.21527719497680664,
       "step": 26,
-      "step_time": 3.80895136899926
+      "step_time": 10.458724108999945
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 63.25,
-      "completions/mean_terminated_length": 63.25,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.0208025872707367,
-      "epoch": 0.21951219512195122,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14226751029491425,
-      "kl": 1.4639559594797902e-05,
-      "learning_rate": 5.621621621621622e-07,
-      "loss": -0.05629514902830124,
-      "num_tokens": 103867.0,
-      "reward": 0.8898874521255493,
-      "reward_std": 0.1414213478565216,
-      "rewards/true_env_reward_fn/mean": 0.8898874521255493,
-      "rewards/true_env_reward_fn/std": 0.1414213478565216,
+      "completions/max_length": 140.0,
+      "completions/max_terminated_length": 140.0,
+      "completions/mean_length": 71.25,
+      "completions/mean_terminated_length": 71.25,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.193794459104538,
+      "epoch": 0.6585365853658537,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07690244168043137,
+      "kl": 1.2164698546257569e-05,
+      "learning_rate": 6.341463414634146e-07,
+      "loss": 0.00818883627653122,
+      "num_tokens": 671153.0,
+      "reward": 0.3635203242301941,
+      "reward_std": 0.23849114775657654,
+      "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+      "rewards/true_env_reward_fn/std": 0.23849113285541534,
       "step": 27,
-      "step_time": 3.227140603999942
+      "step_time": 14.364785926000081
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 66.5,
-      "completions/mean_terminated_length": 66.5,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.278637707233429,
-      "epoch": 0.22764227642276422,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.6319210822694e-05,
-      "kl": 1.394796117892838e-05,
-      "learning_rate": 5.837837837837838e-07,
-      "loss": 6.984611218285863e-07,
-      "num_tokens": 108511.0,
-      "reward": 0.5384680032730103,
-      "reward_std": 0.06977442651987076,
-      "rewards/true_env_reward_fn/mean": 0.5384680032730103,
-      "rewards/true_env_reward_fn/std": 0.06977442651987076,
+      "completions/max_length": 111.0,
+      "completions/max_terminated_length": 111.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.2883787751197815,
+      "epoch": 0.6829268292682927,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0902288407087326,
+      "kl": 1.1798915693361778e-05,
+      "learning_rate": 6.585365853658536e-07,
+      "loss": 0.038317371159791946,
+      "num_tokens": 697614.0,
+      "reward": 0.44166144728660583,
+      "reward_std": 0.25748196244239807,
+      "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+      "rewards/true_env_reward_fn/std": 0.25748199224472046,
       "step": 28,
-      "step_time": 3.3963304120006796
+      "step_time": 10.888908384999922
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/max_terminated_length": 97.0,
-      "completions/mean_length": 73.5,
-      "completions/mean_terminated_length": 73.5,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2547507286071777,
-      "epoch": 0.23577235772357724,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10846269875764847,
-      "kl": 1.5149432329053525e-05,
-      "learning_rate": 6.054054054054054e-07,
-      "loss": 0.004249647259712219,
-      "num_tokens": 111323.0,
-      "reward": 0.6256026029586792,
-      "reward_std": 0.350762277841568,
-      "rewards/true_env_reward_fn/mean": 0.6256026029586792,
-      "rewards/true_env_reward_fn/std": 0.350762277841568,
+      "completions/max_length": 238.0,
+      "completions/max_terminated_length": 238.0,
+      "completions/mean_length": 69.60417175292969,
+      "completions/mean_terminated_length": 69.60417175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3002805709838867,
+      "epoch": 0.7073170731707317,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07522639632225037,
+      "kl": 1.2230455695316778e-05,
+      "learning_rate": 6.829268292682927e-07,
+      "loss": 0.031045034527778625,
+      "num_tokens": 719187.0,
+      "reward": 0.5349087119102478,
+      "reward_std": 0.29909756779670715,
+      "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+      "rewards/true_env_reward_fn/std": 0.29909753799438477,
       "step": 29,
-      "step_time": 4.2103285969988065
+      "step_time": 15.510035302999995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 56.625,
-      "completions/mean_terminated_length": 56.625,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.4687196612358093,
-      "epoch": 0.24390243902439024,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13451272249221802,
-      "kl": 1.3284446140460204e-05,
-      "learning_rate": 6.27027027027027e-07,
-      "loss": 0.05542291700839996,
-      "num_tokens": 115976.0,
-      "reward": 0.3901680111885071,
-      "reward_std": 0.2995865046977997,
-      "rewards/true_env_reward_fn/mean": 0.3901680111885071,
-      "rewards/true_env_reward_fn/std": 0.2995865046977997,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 189.0,
+      "completions/mean_length": 70.91667175292969,
+      "completions/mean_terminated_length": 70.91667175292969,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2718828916549683,
+      "epoch": 0.7317073170731707,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06776711344718933,
+      "kl": 1.2617916354429326e-05,
+      "learning_rate": 7.073170731707316e-07,
+      "loss": 0.09301326423883438,
+      "num_tokens": 744095.0,
+      "reward": 0.43472790718078613,
+      "reward_std": 0.3138841986656189,
+      "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+      "rewards/true_env_reward_fn/std": 0.3138841688632965,
       "step": 30,
-      "step_time": 3.5506420210003853
+      "step_time": 14.50245602599989
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 98.0,
-      "completions/max_terminated_length": 98.0,
-      "completions/mean_length": 76.875,
-      "completions/mean_terminated_length": 76.875,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2640270590782166,
-      "epoch": 0.25203252032520324,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18553969264030457,
-      "kl": 1.2505860468081664e-05,
-      "learning_rate": 6.486486486486486e-07,
-      "loss": -0.015417251735925674,
-      "num_tokens": 118471.0,
-      "reward": 0.6587758660316467,
-      "reward_std": 0.14417217671871185,
-      "rewards/true_env_reward_fn/mean": 0.6587758660316467,
-      "rewards/true_env_reward_fn/std": 0.14417219161987305,
+      "completions/max_length": 153.0,
+      "completions/max_terminated_length": 153.0,
+      "completions/mean_length": 69.77083587646484,
+      "completions/mean_terminated_length": 69.77083587646484,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2918945252895355,
+      "epoch": 0.7560975609756098,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08040682971477509,
+      "kl": 1.2672349157583085e-05,
+      "learning_rate": 7.317073170731707e-07,
+      "loss": 0.0367550291121006,
+      "num_tokens": 764612.0,
+      "reward": 0.5134401321411133,
+      "reward_std": 0.19073942303657532,
+      "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+      "rewards/true_env_reward_fn/std": 0.19073940813541412,
       "step": 31,
-      "step_time": 4.198089399002129
+      "step_time": 11.06186091799998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.5262224078178406,
-      "epoch": 0.2601626016260163,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2357814759016037,
-      "kl": 1.6242850506387185e-05,
-      "learning_rate": 6.702702702702702e-07,
-      "loss": 0.031210914254188538,
-      "num_tokens": 123923.0,
-      "reward": 0.0943702906370163,
-      "reward_std": 0.1497660130262375,
-      "rewards/true_env_reward_fn/mean": 0.0943702906370163,
-      "rewards/true_env_reward_fn/std": 0.1497660130262375,
+      "completions/max_length": 269.0,
+      "completions/max_terminated_length": 269.0,
+      "completions/mean_length": 71.79167175292969,
+      "completions/mean_terminated_length": 71.79167175292969,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "entropy": 1.1679067015647888,
+      "epoch": 0.7804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0744430273771286,
+      "kl": 1.2661263326663175e-05,
+      "learning_rate": 7.560975609756097e-07,
+      "loss": 0.05885648727416992,
+      "num_tokens": 782058.0,
+      "reward": 0.5372593402862549,
+      "reward_std": 0.18350909650325775,
+      "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+      "rewards/true_env_reward_fn/std": 0.18350908160209656,
       "step": 32,
-      "step_time": 3.978757984001277
+      "step_time": 15.808748693000211
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "entropy": 0.9924907088279724,
-      "epoch": 0.2682926829268293,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20393438637256622,
-      "kl": 1.181096149593941e-05,
-      "learning_rate": 6.918918918918919e-07,
-      "loss": -0.0068489015102386475,
-      "num_tokens": 130831.0,
-      "reward": 0.20862048864364624,
-      "reward_std": 0.2418184131383896,
-      "rewards/true_env_reward_fn/mean": 0.20862048864364624,
-      "rewards/true_env_reward_fn/std": 0.2418184131383896,
+      "completions/max_length": 265.0,
+      "completions/max_terminated_length": 265.0,
+      "completions/mean_length": 76.79167175292969,
+      "completions/mean_terminated_length": 76.79167175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.1829756796360016,
+      "epoch": 0.8048780487804879,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.051698025315999985,
+      "kl": 1.0996191576850833e-05,
+      "learning_rate": 7.804878048780488e-07,
+      "loss": 0.010143717750906944,
+      "num_tokens": 810472.0,
+      "reward": 0.4369215667247772,
+      "reward_std": 0.30869919061660767,
+      "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+      "rewards/true_env_reward_fn/std": 0.30869919061660767,
       "step": 33,
-      "step_time": 4.237411461999727
+      "step_time": 24.20358999299981
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 184.0,
-      "completions/max_terminated_length": 184.0,
-      "completions/mean_length": 105.0,
-      "completions/mean_terminated_length": 105.0,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.4909422397613525,
-      "epoch": 0.2764227642276423,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.714608364040032e-05,
-      "kl": 1.3747331649938133e-05,
-      "learning_rate": 7.135135135135134e-07,
-      "loss": 6.856024583612452e-07,
-      "num_tokens": 138663.0,
-      "reward": 0.1821666657924652,
-      "reward_std": 0.2963036298751831,
-      "rewards/true_env_reward_fn/mean": 0.1821666657924652,
-      "rewards/true_env_reward_fn/std": 0.2963036298751831,
+      "completions/max_length": 137.0,
+      "completions/max_terminated_length": 137.0,
+      "completions/mean_length": 61.85416793823242,
+      "completions/mean_terminated_length": 61.85416793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2468958497047424,
+      "epoch": 0.8292682926829268,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09706687182188034,
+      "kl": 1.2097383432774222e-05,
+      "learning_rate": 8.048780487804878e-07,
+      "loss": 0.026558157056570053,
+      "num_tokens": 836713.0,
+      "reward": 0.3587157428264618,
+      "reward_std": 0.2754887044429779,
+      "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+      "rewards/true_env_reward_fn/std": 0.2754887044429779,
       "step": 34,
-      "step_time": 8.45711429900075
+      "step_time": 12.218407348999904
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 61.25,
-      "completions/mean_terminated_length": 61.25,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.0832659006118774,
-      "epoch": 0.2845528455284553,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1297609806060791,
-      "kl": 1.1829738923552213e-05,
-      "learning_rate": 7.351351351351351e-07,
-      "loss": -0.02754262089729309,
-      "num_tokens": 142361.0,
-      "reward": 0.4525124728679657,
-      "reward_std": 0.23157824575901031,
-      "rewards/true_env_reward_fn/mean": 0.4525124728679657,
-      "rewards/true_env_reward_fn/std": 0.2315782606601715,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 59.5625,
+      "completions/mean_terminated_length": 59.5625,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2368170320987701,
+      "epoch": 0.8536585365853658,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08964981138706207,
+      "kl": 1.3131634887031396e-05,
+      "learning_rate": 8.292682926829268e-07,
+      "loss": -0.01139204390347004,
+      "num_tokens": 860028.0,
+      "reward": 0.49109315872192383,
+      "reward_std": 0.20359393954277039,
+      "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+      "rewards/true_env_reward_fn/std": 0.20359393954277039,
       "step": 35,
-      "step_time": 3.564060039998367
+      "step_time": 9.66908789599995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 55.125,
-      "completions/mean_terminated_length": 55.125,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.0677781999111176,
-      "epoch": 0.2926829268292683,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14367543160915375,
-      "kl": 1.2750254427373875e-05,
-      "learning_rate": 7.567567567567568e-07,
-      "loss": -0.001130133867263794,
-      "num_tokens": 145294.0,
-      "reward": 0.6871603727340698,
-      "reward_std": 0.2714426517486572,
-      "rewards/true_env_reward_fn/mean": 0.6871603727340698,
-      "rewards/true_env_reward_fn/std": 0.2714426517486572,
+      "completions/max_length": 102.0,
+      "completions/max_terminated_length": 102.0,
+      "completions/mean_length": 66.02083587646484,
+      "completions/mean_terminated_length": 66.02083587646484,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.1611860394477844,
+      "epoch": 0.8780487804878049,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08851195871829987,
+      "kl": 1.2570341596074286e-05,
+      "learning_rate": 8.536585365853657e-07,
+      "loss": 0.021737128496170044,
+      "num_tokens": 883189.0,
+      "reward": 0.46058258414268494,
+      "reward_std": 0.2632383108139038,
+      "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+      "rewards/true_env_reward_fn/std": 0.2632383108139038,
       "step": 36,
-      "step_time": 3.6285808550001093
+      "step_time": 8.370980583999994
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 104.0,
-      "completions/max_terminated_length": 104.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.408882200717926,
-      "epoch": 0.3008130081300813,
+      "completions/max_length": 135.0,
+      "completions/max_terminated_length": 135.0,
+      "completions/mean_length": 75.58333587646484,
+      "completions/mean_terminated_length": 75.58333587646484,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "entropy": 1.37085822224617,
+      "epoch": 0.9024390243902439,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13866695761680603,
-      "kl": 1.3317891898623202e-05,
-      "learning_rate": 7.783783783783784e-07,
-      "loss": 0.058712199330329895,
-      "num_tokens": 148747.0,
-      "reward": 0.638524055480957,
-      "reward_std": 0.380489706993103,
-      "rewards/true_env_reward_fn/mean": 0.638524055480957,
-      "rewards/true_env_reward_fn/std": 0.3804897367954254,
+      "grad_norm": 0.05852028727531433,
+      "kl": 1.2957561011717189e-05,
+      "learning_rate": 8.780487804878048e-07,
+      "loss": -0.024281952530145645,
+      "num_tokens": 906801.0,
+      "reward": 0.5022324323654175,
+      "reward_std": 0.11637427657842636,
+      "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+      "rewards/true_env_reward_fn/std": 0.11637428402900696,
       "step": 37,
-      "step_time": 4.57648780099953
+      "step_time": 10.285125336999727
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.3680316805839539,
-      "epoch": 0.3089430894308943,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13230997323989868,
-      "kl": 1.1831724805233534e-05,
-      "learning_rate": 8e-07,
-      "loss": -0.06476183235645294,
-      "num_tokens": 152794.0,
-      "reward": 0.47908467054367065,
-      "reward_std": 0.18681679666042328,
-      "rewards/true_env_reward_fn/mean": 0.47908467054367065,
-      "rewards/true_env_reward_fn/std": 0.18681679666042328,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 65.14583587646484,
+      "completions/mean_terminated_length": 65.14583587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2760809361934662,
+      "epoch": 0.926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09105321019887924,
+      "kl": 1.3129126955391257e-05,
+      "learning_rate": 9.024390243902439e-07,
+      "loss": -0.011838603764772415,
+      "num_tokens": 929536.0,
+      "reward": 0.49639374017715454,
+      "reward_std": 0.32166802883148193,
+      "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+      "rewards/true_env_reward_fn/std": 0.32166802883148193,
       "step": 38,
-      "step_time": 3.627890882000429
+      "step_time": 12.449738128000035
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 56.875,
-      "completions/mean_terminated_length": 56.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.3124344944953918,
-      "epoch": 0.3170731707317073,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20907950401306152,
-      "kl": 1.4425510926230345e-05,
-      "learning_rate": 7.999820918660971e-07,
-      "loss": -0.014620006084442139,
-      "num_tokens": 157337.0,
-      "reward": 0.4882892966270447,
-      "reward_std": 0.28137314319610596,
-      "rewards/true_env_reward_fn/mean": 0.4882892966270447,
-      "rewards/true_env_reward_fn/std": 0.28137317299842834,
+      "completions/max_length": 124.0,
+      "completions/max_terminated_length": 124.0,
+      "completions/mean_length": 72.08333587646484,
+      "completions/mean_terminated_length": 72.08333587646484,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.2545586228370667,
+      "epoch": 0.9512195121951219,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06919296830892563,
+      "kl": 1.459557256566768e-05,
+      "learning_rate": 9.26829268292683e-07,
+      "loss": 0.021831180900335312,
+      "num_tokens": 950388.0,
+      "reward": 0.4879913330078125,
+      "reward_std": 0.24854585528373718,
+      "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+      "rewards/true_env_reward_fn/std": 0.24854585528373718,
       "step": 39,
-      "step_time": 3.5362214279994078
+      "step_time": 10.279209028999958
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 51.625,
-      "completions/mean_terminated_length": 51.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 0.9928885996341705,
-      "epoch": 0.3252032520325203,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.641438762424514e-05,
-      "kl": 1.1731265658454504e-05,
-      "learning_rate": 7.99928369067895e-07,
-      "loss": 5.910313234380737e-07,
-      "num_tokens": 160166.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/max_length": 179.0,
+      "completions/max_terminated_length": 179.0,
+      "completions/mean_length": 74.20833587646484,
+      "completions/mean_terminated_length": 74.20833587646484,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2255937159061432,
+      "epoch": 0.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06352153420448303,
+      "kl": 1.2041192348988261e-05,
+      "learning_rate": 9.512195121951218e-07,
+      "loss": -0.013997981324791908,
+      "num_tokens": 981254.0,
+      "reward": 0.39802420139312744,
+      "reward_std": 0.20212584733963013,
+      "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+      "rewards/true_env_reward_fn/std": 0.20212584733963013,
       "step": 40,
-      "step_time": 3.1957039770022675
+      "step_time": 13.58010066599968
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 54.75,
-      "completions/mean_terminated_length": 54.75,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.2997828722000122,
-      "epoch": 0.3333333333333333,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21271590888500214,
-      "kl": 1.3209032658778597e-05,
-      "learning_rate": 7.99838836415769e-07,
-      "loss": 0.033298641443252563,
-      "num_tokens": 165884.0,
-      "reward": 0.2860966920852661,
-      "reward_std": 0.2721884846687317,
-      "rewards/true_env_reward_fn/mean": 0.2860966920852661,
-      "rewards/true_env_reward_fn/std": 0.2721884846687317,
+      "completions/max_length": 139.0,
+      "completions/max_terminated_length": 139.0,
+      "completions/mean_length": 75.04167175292969,
+      "completions/mean_terminated_length": 75.04167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.2703719735145569,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.045169439166784286,
+      "kl": 1.1270850109212915e-05,
+      "learning_rate": 9.756097560975609e-07,
+      "loss": -0.010194316506385803,
+      "num_tokens": 1009968.0,
+      "reward": 0.4517599940299988,
+      "reward_std": 0.11791092902421951,
+      "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+      "rewards/true_env_reward_fn/std": 0.11791091412305832,
       "step": 41,
-      "step_time": 3.6851942720004445
+      "step_time": 10.35077203700007
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 48.875,
-      "completions/mean_terminated_length": 48.875,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.4380556344985962,
-      "epoch": 0.34146341463414637,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.17314757406711578,
-      "kl": 9.354779194836738e-06,
-      "learning_rate": 7.997135019265325e-07,
-      "loss": 0.08398272097110748,
-      "num_tokens": 172067.0,
-      "reward": -0.003943998366594315,
-      "reward_std": 0.13122709095478058,
-      "rewards/true_env_reward_fn/mean": -0.003943998366594315,
-      "rewards/true_env_reward_fn/std": 0.13122709095478058,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 64.33333587646484,
+      "completions/mean_terminated_length": 64.33333587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.329576164484024,
+      "epoch": 1.024390243902439,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08522730320692062,
+      "kl": 1.4469044799625408e-05,
+      "learning_rate": 1e-06,
+      "loss": -0.00014946190640330315,
+      "num_tokens": 1039032.0,
+      "reward": 0.33548423647880554,
+      "reward_std": 0.22271563112735748,
+      "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+      "rewards/true_env_reward_fn/std": 0.22271563112735748,
       "step": 42,
-      "step_time": 3.545334507000007
+      "step_time": 10.548370664999993
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.0,
-      "completions/max_terminated_length": 123.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 71.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.163844645023346,
-      "epoch": 0.34959349593495936,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15747681260108948,
-      "kl": 1.3550960375141585e-05,
-      "learning_rate": 7.995523768227198e-07,
-      "loss": 0.05901219695806503,
-      "num_tokens": 176427.0,
-      "reward": 0.3297747075557709,
-      "reward_std": 0.4647028148174286,
-      "rewards/true_env_reward_fn/mean": 0.3297747075557709,
-      "rewards/true_env_reward_fn/std": 0.464702844619751,
+      "completions/max_length": 372.0,
+      "completions/max_terminated_length": 372.0,
+      "completions/mean_length": 70.02083587646484,
+      "completions/mean_terminated_length": 70.02083587646484,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.2357364892959595,
+      "epoch": 1.048780487804878,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07030358910560608,
+      "kl": 1.3562755839302554e-05,
+      "learning_rate": 9.999818789066163e-07,
+      "loss": -0.02616041898727417,
+      "num_tokens": 1060833.0,
+      "reward": 0.5167371034622192,
+      "reward_std": 0.24280032515525818,
+      "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+      "rewards/true_env_reward_fn/std": 0.24280032515525818,
       "step": 43,
-      "step_time": 5.4708715960005065
+      "step_time": 24.089396637999698
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.3323996663093567,
-      "epoch": 0.35772357723577236,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15227818489074707,
-      "kl": 1.1237668786634458e-05,
-      "learning_rate": 7.993554755315805e-07,
-      "loss": 0.0660967156291008,
-      "num_tokens": 181912.0,
-      "reward": 0.22226500511169434,
-      "reward_std": 0.2765512466430664,
-      "rewards/true_env_reward_fn/mean": 0.22226500511169434,
-      "rewards/true_env_reward_fn/std": 0.2765512466430664,
+      "completions/max_length": 234.0,
+      "completions/max_terminated_length": 234.0,
+      "completions/mean_length": 77.47917175292969,
+      "completions/mean_terminated_length": 77.47917175292969,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 1.1693778038024902,
+      "epoch": 1.0731707317073171,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07017157226800919,
+      "kl": 1.332453393843025e-05,
+      "learning_rate": 9.999275169399612e-07,
+      "loss": -0.006466507911682129,
+      "num_tokens": 1088648.0,
+      "reward": 0.4498252272605896,
+      "reward_std": 0.21398545801639557,
+      "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+      "rewards/true_env_reward_fn/std": 0.21398545801639557,
       "step": 44,
-      "step_time": 3.940563359999942
+      "step_time": 19.39071501599983
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 56.375,
-      "completions/mean_terminated_length": 56.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.241302490234375,
-      "epoch": 0.36585365853658536,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1711702048778534,
-      "kl": 1.1479866316221887e-05,
-      "learning_rate": 7.991228156837879e-07,
-      "loss": 0.0959811806678772,
-      "num_tokens": 186099.0,
-      "reward": 0.4569639563560486,
-      "reward_std": 0.356449693441391,
-      "rewards/true_env_reward_fn/mean": 0.4569639563560486,
-      "rewards/true_env_reward_fn/std": 0.356449693441391,
+      "completions/max_length": 186.0,
+      "completions/max_terminated_length": 186.0,
+      "completions/mean_length": 72.16667175292969,
+      "completions/mean_terminated_length": 72.16667175292969,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3268415927886963,
+      "epoch": 1.0975609756097562,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06632921099662781,
+      "kl": 1.4458733630817733e-05,
+      "learning_rate": 9.99836918040428e-07,
+      "loss": -0.03534461930394173,
+      "num_tokens": 1117096.0,
+      "reward": 0.4053138196468353,
+      "reward_std": 0.21476909518241882,
+      "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+      "rewards/true_env_reward_fn/std": 0.21476909518241882,
       "step": 45,
-      "step_time": 3.947248132999448
+      "step_time": 13.893569495999827
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 66.625,
-      "completions/mean_terminated_length": 66.625,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.5153677463531494,
-      "epoch": 0.37398373983739835,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22329360246658325,
-      "kl": 1.3615897842100821e-05,
-      "learning_rate": 7.988544181118608e-07,
-      "loss": 0.07407481223344803,
-      "num_tokens": 192056.0,
-      "reward": 0.2950569987297058,
-      "reward_std": 0.2872281074523926,
-      "rewards/true_env_reward_fn/mean": 0.2950569987297058,
-      "rewards/true_env_reward_fn/std": 0.28722813725471497,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 70.16667175292969,
+      "completions/mean_terminated_length": 70.16667175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2670875787734985,
+      "epoch": 1.1219512195121952,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08321154117584229,
+      "kl": 1.4837954950053245e-05,
+      "learning_rate": 9.997100887750215e-07,
+      "loss": -0.039235007017850876,
+      "num_tokens": 1136480.0,
+      "reward": 0.48141974210739136,
+      "reward_std": 0.2837103307247162,
+      "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+      "rewards/true_env_reward_fn/std": 0.2837103009223938,
       "step": 46,
-      "step_time": 4.1211709569997765
+      "step_time": 10.50698806499986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.105223298072815,
-      "epoch": 0.3821138211382114,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1306377500295639,
-      "kl": 1.2826244528696407e-05,
-      "learning_rate": 7.985503068482974e-07,
-      "loss": 0.014609627425670624,
-      "num_tokens": 195544.0,
-      "reward": 0.5289265513420105,
-      "reward_std": 0.3883950710296631,
-      "rewards/true_env_reward_fn/mean": 0.5289265513420105,
-      "rewards/true_env_reward_fn/std": 0.3883951008319855,
+      "completions/max_length": 159.0,
+      "completions/max_terminated_length": 159.0,
+      "completions/mean_length": 76.1875,
+      "completions/mean_terminated_length": 76.1875,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3037313222885132,
+      "epoch": 1.146341463414634,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.061912886798381805,
+      "kl": 1.283655774386716e-05,
+      "learning_rate": 9.995470383368808e-07,
+      "loss": -0.01992109790444374,
+      "num_tokens": 1162249.0,
+      "reward": 0.49922606348991394,
+      "reward_std": 0.2621309757232666,
+      "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+      "rewards/true_env_reward_fn/std": 0.2621309757232666,
       "step": 47,
-      "step_time": 2.938600743000279
+      "step_time": 12.964419044000124
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 67.375,
-      "completions/mean_terminated_length": 67.375,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.5243317484855652,
-      "epoch": 0.3902439024390244,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20161111652851105,
-      "kl": 1.4497059055429418e-05,
-      "learning_rate": 7.982105091234235e-07,
-      "loss": 0.23342597484588623,
-      "num_tokens": 198691.0,
-      "reward": 0.45001715421676636,
-      "reward_std": 0.2565726041793823,
-      "rewards/true_env_reward_fn/mean": 0.45001715421676636,
-      "rewards/true_env_reward_fn/std": 0.2565726041793823,
+      "completions/max_length": 231.0,
+      "completions/max_terminated_length": 231.0,
+      "completions/mean_length": 71.375,
+      "completions/mean_terminated_length": 71.375,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2007178366184235,
+      "epoch": 1.170731707317073,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0889662653207779,
+      "kl": 1.6228528693318367e-05,
+      "learning_rate": 9.993477785446149e-07,
+      "loss": 0.045945264399051666,
+      "num_tokens": 1184555.0,
+      "reward": 0.42501482367515564,
+      "reward_std": 0.27350595593452454,
+      "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+      "rewards/true_env_reward_fn/std": 0.27350592613220215,
       "step": 48,
-      "step_time": 4.91795033499875
+      "step_time": 17.23041258299986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 55.375,
-      "completions/mean_terminated_length": 55.375,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2544435858726501,
-      "epoch": 0.3983739837398374,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11793916672468185,
-      "kl": 1.3676196886081016e-05,
-      "learning_rate": 7.978350553629554e-07,
-      "loss": -0.016418367624282837,
-      "num_tokens": 202994.0,
-      "reward": 0.4054500162601471,
-      "reward_std": 0.20634961128234863,
-      "rewards/true_env_reward_fn/mean": 0.4054500162601471,
-      "rewards/true_env_reward_fn/std": 0.20634961128234863,
+      "completions/max_length": 86.0,
+      "completions/max_terminated_length": 86.0,
+      "completions/mean_length": 55.9375,
+      "completions/mean_terminated_length": 55.9375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.182040810585022,
+      "epoch": 1.1951219512195121,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08547856658697128,
+      "kl": 1.571832831359643e-05,
+      "learning_rate": 9.991123238414453e-07,
+      "loss": 0.02548346482217312,
+      "num_tokens": 1208384.0,
+      "reward": 0.3845663070678711,
+      "reward_std": 0.315467894077301,
+      "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+      "rewards/true_env_reward_fn/std": 0.31546786427497864,
       "step": 49,
-      "step_time": 3.626596234000317
+      "step_time": 8.691208415999881
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 62.5,
-      "completions/mean_terminated_length": 62.5,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.022342562675476,
-      "epoch": 0.4065040650406504,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16596083343029022,
-      "kl": 1.1194244052603608e-05,
-      "learning_rate": 7.974239791852739e-07,
-      "loss": 0.0499756895005703,
-      "num_tokens": 205770.0,
-      "reward": 0.5639185309410095,
-      "reward_std": 0.1721728891134262,
-      "rewards/true_env_reward_fn/mean": 0.5639185309410095,
-      "rewards/true_env_reward_fn/std": 0.1721728891134262,
+      "completions/max_length": 109.0,
+      "completions/max_terminated_length": 109.0,
+      "completions/mean_length": 64.75,
+      "completions/mean_terminated_length": 64.75,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2306177020072937,
+      "epoch": 1.2195121951219512,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07395736873149872,
+      "kl": 1.2643881973417592e-05,
+      "learning_rate": 9.988406912941589e-07,
+      "loss": -0.04186868295073509,
+      "num_tokens": 1227700.0,
+      "reward": 0.5068289637565613,
+      "reward_std": 0.31324177980422974,
+      "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+      "rewards/true_env_reward_fn/std": 0.31324175000190735,
       "step": 50,
-      "step_time": 3.9679293660010444
+      "step_time": 10.162109979000206
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 60.625,
-      "completions/mean_terminated_length": 60.625,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 0.9613964557647705,
-      "epoch": 0.4146341463414634,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12812700867652893,
-      "kl": 1.0464088063599775e-05,
-      "learning_rate": 7.969773173984153e-07,
-      "loss": 0.023206032812595367,
-      "num_tokens": 210443.0,
-      "reward": 0.3208000063896179,
-      "reward_std": 0.25050169229507446,
-      "rewards/true_env_reward_fn/mean": 0.3208000063896179,
-      "rewards/true_env_reward_fn/std": 0.25050172209739685,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 61.25,
+      "completions/mean_terminated_length": 61.25,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 1.2760189771652222,
+      "epoch": 1.2439024390243902,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0870513767004013,
+      "kl": 1.4371181578098913e-05,
+      "learning_rate": 9.985329005918702e-07,
+      "loss": -0.01623840071260929,
+      "num_tokens": 1253120.0,
+      "reward": 0.3888077139854431,
+      "reward_std": 0.3346175253391266,
+      "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+      "rewards/true_env_reward_fn/std": 0.3346175253391266,
       "step": 51,
-      "step_time": 3.6275602460009395
+      "step_time": 10.88732858200001
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.2359730005264282,
-      "epoch": 0.42276422764227645,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1384950578212738,
-      "kl": 1.2094554222130682e-05,
-      "learning_rate": 7.964951099967749e-07,
-      "loss": -0.07054222375154495,
-      "num_tokens": 213833.0,
-      "reward": 0.5900156497955322,
-      "reward_std": 0.18237514793872833,
-      "rewards/true_env_reward_fn/mean": 0.5900156497955322,
-      "rewards/true_env_reward_fn/std": 0.18237514793872833,
+      "completions/max_length": 93.0,
+      "completions/max_terminated_length": 93.0,
+      "completions/mean_length": 52.72916793823242,
+      "completions/mean_terminated_length": 52.72916793823242,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.1669773161411285,
+      "epoch": 1.2682926829268293,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.1055479422211647,
+      "kl": 1.69047059443983e-05,
+      "learning_rate": 9.981889740445957e-07,
+      "loss": 0.03519687056541443,
+      "num_tokens": 1274803.0,
+      "reward": 0.4995749592781067,
+      "reward_std": 0.2088174670934677,
+      "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+      "rewards/true_env_reward_fn/std": 0.2088174819946289,
       "step": 52,
-      "step_time": 3.8849526029989647
+      "step_time": 9.252596976000177
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 79.5,
-      "completions/mean_terminated_length": 79.5,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.2706108689308167,
-      "epoch": 0.43089430894308944,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.17388403415679932,
-      "kl": 1.3583369309344562e-05,
-      "learning_rate": 7.959774001575264e-07,
-      "loss": 0.06114684417843819,
-      "num_tokens": 216853.0,
-      "reward": 0.4848448634147644,
-      "reward_std": 0.2859330177307129,
-      "rewards/true_env_reward_fn/mean": 0.4848448634147644,
-      "rewards/true_env_reward_fn/std": 0.2859330177307129,
+      "completions/max_length": 310.0,
+      "completions/max_terminated_length": 310.0,
+      "completions/mean_length": 76.72917175292969,
+      "completions/mean_terminated_length": 76.72917175292969,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2451297044754028,
+      "epoch": 1.2926829268292683,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07513763755559921,
+      "kl": 1.5911174841676257e-05,
+      "learning_rate": 9.978089365816355e-07,
+      "loss": 0.07006432861089706,
+      "num_tokens": 1305910.0,
+      "reward": 0.33895593881607056,
+      "reward_std": 0.2969740927219391,
+      "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+      "rewards/true_env_reward_fn/std": 0.2969740927219391,
       "step": 53,
-      "step_time": 4.964324356000361
+      "step_time": 24.22518693999973
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 64.875,
-      "completions/mean_terminated_length": 64.875,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2430712580680847,
-      "epoch": 0.43902439024390244,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11113106459379196,
-      "kl": 1.2204414360894589e-05,
-      "learning_rate": 7.954242342367553e-07,
-      "loss": 0.010590985417366028,
-      "num_tokens": 221252.0,
-      "reward": 0.392258882522583,
-      "reward_std": 0.13280020654201508,
-      "rewards/true_env_reward_fn/mean": 0.392258882522583,
-      "rewards/true_env_reward_fn/std": 0.13280019164085388,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 66.02083587646484,
+      "completions/mean_terminated_length": 66.02083587646484,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.3110129833221436,
+      "epoch": 1.3170731707317074,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0695035383105278,
+      "kl": 1.606306568646687e-05,
+      "learning_rate": 9.973928157497674e-07,
+      "loss": 0.03299739956855774,
+      "num_tokens": 1330815.0,
+      "reward": 0.4440445899963379,
+      "reward_std": 0.2889502942562103,
+      "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+      "rewards/true_env_reward_fn/std": 0.2889502942562103,
       "step": 54,
-      "step_time": 3.5511989209990134
+      "step_time": 10.14821418500037
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 118.0,
-      "completions/max_terminated_length": 118.0,
-      "completions/mean_length": 61.875,
-      "completions/mean_terminated_length": 61.875,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.3175880908966064,
-      "epoch": 0.44715447154471544,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20881597697734833,
-      "kl": 1.58558846123924e-05,
-      "learning_rate": 7.948356617653087e-07,
-      "loss": -0.06772151589393616,
-      "num_tokens": 224691.0,
-      "reward": 0.30961817502975464,
-      "reward_std": 0.27422165870666504,
-      "rewards/true_env_reward_fn/mean": 0.30961817502975464,
-      "rewards/true_env_reward_fn/std": 0.27422165870666504,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 61.958335876464844,
+      "completions/mean_terminated_length": 61.958335876464844,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2969173192977905,
+      "epoch": 1.3414634146341464,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09475436061620712,
+      "kl": 1.5850717545617954e-05,
+      "learning_rate": 9.969406417112488e-07,
+      "loss": -0.014009319245815277,
+      "num_tokens": 1361885.0,
+      "reward": 0.3289160430431366,
+      "reward_std": 0.26591774821281433,
+      "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+      "rewards/true_env_reward_fn/std": 0.2659177780151367,
       "step": 55,
-      "step_time": 5.031640098000935
+      "step_time": 11.13082981000025
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 64.625,
-      "completions/mean_terminated_length": 64.625,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.4056915640830994,
-      "epoch": 0.45528455284552843,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.606108895037323e-05,
-      "kl": 1.2847603557020193e-05,
-      "learning_rate": 7.942117354443597e-07,
-      "loss": 6.408997705875663e-07,
-      "num_tokens": 228116.0,
-      "reward": 0.5599601864814758,
-      "reward_std": 0.18355616927146912,
-      "rewards/true_env_reward_fn/mean": 0.5599601864814758,
-      "rewards/true_env_reward_fn/std": 0.18355616927146912,
+      "completions/max_length": 154.0,
+      "completions/max_terminated_length": 154.0,
+      "completions/mean_length": 80.22917175292969,
+      "completions/mean_terminated_length": 80.22917175292969,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.2630544006824493,
+      "epoch": 1.3658536585365852,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.06734384596347809,
+      "kl": 1.5091616887730197e-05,
+      "learning_rate": 9.964524472416317e-07,
+      "loss": -0.10958556830883026,
+      "num_tokens": 1390496.0,
+      "reward": 0.46485185623168945,
+      "reward_std": 0.29441413283348083,
+      "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+      "rewards/true_env_reward_fn/std": 0.29441413283348083,
       "step": 56,
-      "step_time": 3.6221305880008003
+      "step_time": 14.49393488900023
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 59.875,
-      "completions/mean_terminated_length": 59.875,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.4034882187843323,
-      "epoch": 0.4634146341463415,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19157616794109344,
-      "kl": 1.4551038475474343e-05,
-      "learning_rate": 7.935525111406885e-07,
-      "loss": 0.021202675998210907,
-      "num_tokens": 233139.0,
-      "reward": 0.32785865664482117,
-      "reward_std": 0.2835054397583008,
-      "rewards/true_env_reward_fn/mean": 0.32785865664482117,
-      "rewards/true_env_reward_fn/std": 0.28350546956062317,
+      "completions/max_length": 203.0,
+      "completions/max_terminated_length": 203.0,
+      "completions/mean_length": 64.27083587646484,
+      "completions/mean_terminated_length": 64.27083587646484,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.1640427708625793,
+      "epoch": 1.3902439024390243,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.07240130007266998,
+      "kl": 1.509602225269191e-05,
+      "learning_rate": 9.959282677273868e-07,
+      "loss": 0.10520926117897034,
+      "num_tokens": 1411837.0,
+      "reward": 0.5296112895011902,
+      "reward_std": 0.2505757212638855,
+      "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+      "rewards/true_env_reward_fn/std": 0.2505757212638855,
       "step": 57,
-      "step_time": 3.7005361410010664
+      "step_time": 14.065935286000013
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 173.0,
-      "completions/max_terminated_length": 173.0,
-      "completions/mean_length": 76.625,
-      "completions/mean_terminated_length": 76.625,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.2941595911979675,
-      "epoch": 0.4715447154471545,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 65.58333587646484,
+      "completions/mean_terminated_length": 65.58333587646484,
+      "completions/min_length": 28.0,
+      "completions/min_terminated_length": 28.0,
+      "entropy": 1.1222519278526306,
+      "epoch": 1.4146341463414633,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14299193024635315,
-      "kl": 1.3164159554435173e-05,
-      "learning_rate": 7.92858047881681e-07,
-      "loss": -0.14726585149765015,
-      "num_tokens": 238584.0,
-      "reward": 0.444433331489563,
-      "reward_std": 0.030650291591882706,
-      "rewards/true_env_reward_fn/mean": 0.444433331489563,
-      "rewards/true_env_reward_fn/std": 0.030650287866592407,
+      "grad_norm": 0.05992415174841881,
+      "kl": 1.2099166724510724e-05,
+      "learning_rate": 9.953681411633374e-07,
+      "loss": 0.004622246604412794,
+      "num_tokens": 1438569.0,
+      "reward": 0.41778087615966797,
+      "reward_std": 0.28395572304725647,
+      "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+      "rewards/true_env_reward_fn/std": 0.2839556932449341,
       "step": 58,
-      "step_time": 7.550715425000817
+      "step_time": 12.76291504100027
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 54.625,
-      "completions/mean_terminated_length": 54.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1917714476585388,
-      "epoch": 0.4796747967479675,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.25083038210868835,
-      "kl": 1.3176229913369752e-05,
-      "learning_rate": 7.921284078500422e-07,
-      "loss": 0.088463693857193,
-      "num_tokens": 240669.0,
-      "reward": 0.7982887029647827,
-      "reward_std": 0.1672983169555664,
-      "rewards/true_env_reward_fn/mean": 0.7982887029647827,
-      "rewards/true_env_reward_fn/std": 0.1672983169555664,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 75.33333587646484,
+      "completions/mean_terminated_length": 75.33333587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.1821540892124176,
+      "epoch": 1.4390243902439024,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0654020607471466,
+      "kl": 1.4932766589481616e-05,
+      "learning_rate": 9.947721081499067e-07,
+      "loss": 0.06719422340393066,
+      "num_tokens": 1461033.0,
+      "reward": 0.5268458127975464,
+      "reward_std": 0.23783695697784424,
+      "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+      "rewards/true_env_reward_fn/std": 0.23783694207668304,
       "step": 59,
-      "step_time": 3.7769912429994292
+      "step_time": 11.089177285999767
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 66.375,
-      "completions/mean_terminated_length": 66.375,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.3743653893470764,
-      "epoch": 0.4878048780487805,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18919643759727478,
-      "kl": 1.231462101713987e-05,
-      "learning_rate": 7.91363656378229e-07,
-      "loss": -0.08548973500728607,
-      "num_tokens": 243808.0,
-      "reward": 0.5988538861274719,
-      "reward_std": 0.11870570480823517,
-      "rewards/true_env_reward_fn/mean": 0.5988538861274719,
-      "rewards/true_env_reward_fn/std": 0.11870571970939636,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 72.14583587646484,
+      "completions/mean_terminated_length": 72.14583587646484,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2446446418762207,
+      "epoch": 1.4634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08000538498163223,
+      "kl": 1.3416995898296591e-05,
+      "learning_rate": 9.941402118901742e-07,
+      "loss": 0.05287330970168114,
+      "num_tokens": 1488264.0,
+      "reward": 0.4032561779022217,
+      "reward_std": 0.24067741632461548,
+      "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+      "rewards/true_env_reward_fn/std": 0.24067740142345428,
       "step": 60,
-      "step_time": 4.052767743998629
+      "step_time": 12.328215124000053
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 132.0,
-      "completions/max_terminated_length": 132.0,
-      "completions/mean_length": 77.125,
-      "completions/mean_terminated_length": 77.125,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.329764723777771,
-      "epoch": 0.4959349593495935,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1690233051776886,
-      "kl": 1.405783814334427e-05,
-      "learning_rate": 7.905638619426003e-07,
-      "loss": 0.0050433604046702385,
-      "num_tokens": 248725.0,
-      "reward": 0.27516257762908936,
-      "reward_std": 0.32322537899017334,
-      "rewards/true_env_reward_fn/mean": 0.27516257762908936,
-      "rewards/true_env_reward_fn/std": 0.32322537899017334,
+      "completions/max_length": 191.0,
+      "completions/max_terminated_length": 191.0,
+      "completions/mean_length": 65.8125,
+      "completions/mean_terminated_length": 65.8125,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.416578859090805,
+      "epoch": 1.4878048780487805,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07916785031557083,
+      "kl": 1.8312134670850355e-05,
+      "learning_rate": 9.934724981867446e-07,
+      "loss": -0.02956264466047287,
+      "num_tokens": 1506607.0,
+      "reward": 0.6846215724945068,
+      "reward_std": 0.21603551506996155,
+      "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+      "rewards/true_env_reward_fn/std": 0.21603552997112274,
       "step": 61,
-      "step_time": 6.010593229999358
+      "step_time": 13.294195681999781
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 50.0,
-      "completions/min_terminated_length": 50.0,
-      "entropy": 1.2542970776557922,
-      "epoch": 0.5040650406504065,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11462891101837158,
-      "kl": 1.13775058707688e-05,
-      "learning_rate": 7.897290961572853e-07,
-      "loss": -0.007184989750385284,
-      "num_tokens": 252101.0,
-      "reward": 0.5372380018234253,
-      "reward_std": 0.13500821590423584,
-      "rewards/true_env_reward_fn/mean": 0.5372380018234253,
-      "rewards/true_env_reward_fn/std": 0.13500821590423584,
+      "completions/max_length": 182.0,
+      "completions/max_terminated_length": 182.0,
+      "completions/mean_length": 64.33333587646484,
+      "completions/mean_terminated_length": 64.33333587646484,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2478635609149933,
+      "epoch": 1.5121951219512195,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09035050123929977,
+      "kl": 1.867344440142915e-05,
+      "learning_rate": 9.927690154384272e-07,
+      "loss": -0.048415351659059525,
+      "num_tokens": 1539351.0,
+      "reward": 0.17628252506256104,
+      "reward_std": 0.2993278205394745,
+      "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+      "rewards/true_env_reward_fn/std": 0.2993278503417969,
       "step": 62,
-      "step_time": 3.4512634010006877
+      "step_time": 18.972790307999958
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 65.75,
-      "completions/mean_terminated_length": 65.75,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.1982964873313904,
-      "epoch": 0.5121951219512195,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12309258431196213,
-      "kl": 1.69004347299051e-05,
-      "learning_rate": 7.888594337677712e-07,
-      "loss": 0.0009508281946182251,
-      "num_tokens": 255231.0,
-      "reward": 0.6114543080329895,
-      "reward_std": 0.10413603484630585,
-      "rewards/true_env_reward_fn/mean": 0.6114543080329895,
-      "rewards/true_env_reward_fn/std": 0.10413603484630585,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 61.66666793823242,
+      "completions/mean_terminated_length": 61.66666793823242,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2024545669555664,
+      "epoch": 1.5365853658536586,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09254598617553711,
+      "kl": 1.654068455536617e-05,
+      "learning_rate": 9.920298146367286e-07,
+      "loss": 0.09414710104465485,
+      "num_tokens": 1565215.0,
+      "reward": 0.4147046208381653,
+      "reward_std": 0.1770697683095932,
+      "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+      "rewards/true_env_reward_fn/std": 0.1770697683095932,
       "step": 63,
-      "step_time": 3.735559521997857
+      "step_time": 11.292631820999986
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 129.0,
-      "completions/max_terminated_length": 129.0,
-      "completions/mean_length": 69.0,
-      "completions/mean_terminated_length": 69.0,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.3587612509727478,
-      "epoch": 0.5203252032520326,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15858450531959534,
-      "kl": 1.4598341294913553e-05,
-      "learning_rate": 7.879549526442108e-07,
-      "loss": 0.0696716383099556,
-      "num_tokens": 260523.0,
-      "reward": 0.2912999987602234,
-      "reward_std": 0.2844822406768799,
-      "rewards/true_env_reward_fn/mean": 0.2912999987602234,
-      "rewards/true_env_reward_fn/std": 0.2844822406768799,
+      "completions/max_length": 120.0,
+      "completions/max_terminated_length": 120.0,
+      "completions/mean_length": 64.1875,
+      "completions/mean_terminated_length": 64.1875,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2442612051963806,
+      "epoch": 1.5609756097560976,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.052295491099357605,
+      "kl": 1.716045289867907e-05,
+      "learning_rate": 9.912549493621554e-07,
+      "loss": 0.01475335843861103,
+      "num_tokens": 1589608.0,
+      "reward": 0.48741206526756287,
+      "reward_std": 0.21404753625392914,
+      "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+      "rewards/true_env_reward_fn/std": 0.21404753625392914,
       "step": 64,
-      "step_time": 5.731267729999672
+      "step_time": 9.600786530999585
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 230.0,
-      "completions/max_terminated_length": 230.0,
-      "completions/mean_length": 83.5,
-      "completions/mean_terminated_length": 83.5,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2284430861473083,
-      "epoch": 0.5284552845528455,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13374178111553192,
-      "kl": 1.2341822639427846e-05,
-      "learning_rate": 7.870157337744494e-07,
-      "loss": 0.10693901032209396,
-      "num_tokens": 264967.0,
-      "reward": 0.3284733295440674,
-      "reward_std": 0.3848404288291931,
-      "rewards/true_env_reward_fn/mean": 0.3284733295440674,
-      "rewards/true_env_reward_fn/std": 0.3848404288291931,
+      "completions/max_length": 111.0,
+      "completions/max_terminated_length": 111.0,
+      "completions/mean_length": 61.3125,
+      "completions/mean_terminated_length": 61.3125,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.2735203504562378,
+      "epoch": 1.5853658536585367,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.094636932015419,
+      "kl": 2.017962560785236e-05,
+      "learning_rate": 9.90444475780332e-07,
+      "loss": -0.0016674790531396866,
+      "num_tokens": 1619095.0,
+      "reward": 0.34675830602645874,
+      "reward_std": 0.2556215822696686,
+      "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+      "rewards/true_env_reward_fn/std": 0.2556215822696686,
       "step": 65,
-      "step_time": 9.601442954000959
+      "step_time": 10.169144185999812
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 132.0,
-      "completions/max_terminated_length": 132.0,
-      "completions/mean_length": 85.25,
-      "completions/mean_terminated_length": 85.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2396279573440552,
-      "epoch": 0.5365853658536586,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.08427461981773376,
-      "kl": 1.4658115105703473e-05,
-      "learning_rate": 7.860418612567733e-07,
-      "loss": -0.05642998591065407,
-      "num_tokens": 269717.0,
-      "reward": 0.38946664333343506,
-      "reward_std": 0.1897086799144745,
-      "rewards/true_env_reward_fn/mean": 0.38946664333343506,
-      "rewards/true_env_reward_fn/std": 0.1897086799144745,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 65.9375,
+      "completions/mean_terminated_length": 65.9375,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2845994234085083,
+      "epoch": 1.6097560975609757,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05164559930562973,
+      "kl": 1.66792838172114e-05,
+      "learning_rate": 9.89598452637928e-07,
+      "loss": 0.011961851269006729,
+      "num_tokens": 1645076.0,
+      "reward": 0.44275379180908203,
+      "reward_std": 0.2063576877117157,
+      "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+      "rewards/true_env_reward_fn/std": 0.2063576877117157,
       "step": 66,
-      "step_time": 6.017849919000582
+      "step_time": 10.343706631000032
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 72.25,
-      "completions/mean_terminated_length": 72.25,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.2245049476623535,
-      "epoch": 0.5447154471544715,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13357418775558472,
-      "kl": 1.2806529412046075e-05,
-      "learning_rate": 7.850334222923798e-07,
-      "loss": 0.03744228184223175,
-      "num_tokens": 275407.0,
-      "reward": 0.08966667205095291,
-      "reward_std": 0.23612774908542633,
-      "rewards/true_env_reward_fn/mean": 0.08966667205095291,
-      "rewards/true_env_reward_fn/std": 0.23612776398658752,
+      "completions/max_length": 141.0,
+      "completions/max_terminated_length": 141.0,
+      "completions/mean_length": 66.0625,
+      "completions/mean_terminated_length": 66.0625,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3418844938278198,
+      "epoch": 1.6341463414634148,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05729615315794945,
+      "kl": 1.6437259546364658e-05,
+      "learning_rate": 9.88716941258401e-07,
+      "loss": 0.015346314758062363,
+      "num_tokens": 1675423.0,
+      "reward": 0.4190921187400818,
+      "reward_std": 0.3388116955757141,
+      "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+      "rewards/true_env_reward_fn/std": 0.3388116657733917,
       "step": 67,
-      "step_time": 4.4363536659984675
+      "step_time": 12.16719347000003
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 56.875,
-      "completions/mean_terminated_length": 56.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.2679882645606995,
-      "epoch": 0.5528455284552846,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15667268633842468,
-      "kl": 1.2213955869810889e-05,
-      "learning_rate": 7.83990507177569e-07,
-      "loss": -0.052396662533283234,
-      "num_tokens": 280838.0,
-      "reward": 0.2431039959192276,
-      "reward_std": 0.2672288715839386,
-      "rewards/true_env_reward_fn/mean": 0.2431039959192276,
-      "rewards/true_env_reward_fn/std": 0.2672288715839386,
+      "completions/max_length": 230.0,
+      "completions/max_terminated_length": 230.0,
+      "completions/mean_length": 78.66667175292969,
+      "completions/mean_terminated_length": 78.66667175292969,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.2716343402862549,
+      "epoch": 1.6585365853658538,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07959166914224625,
+      "kl": 1.785568679224525e-05,
+      "learning_rate": 9.87800005537551e-07,
+      "loss": 0.08119910955429077,
+      "num_tokens": 1698103.0,
+      "reward": 0.5178458094596863,
+      "reward_std": 0.22635267674922943,
+      "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+      "rewards/true_env_reward_fn/std": 0.22635267674922943,
       "step": 68,
-      "step_time": 3.6370441849994677
+      "step_time": 15.523659553000016
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 78.0,
-      "completions/max_terminated_length": 78.0,
-      "completions/mean_length": 62.625,
-      "completions/mean_terminated_length": 62.625,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.2563416361808777,
-      "epoch": 0.5609756097560976,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.937557868193835e-05,
-      "kl": 1.1138304216729011e-05,
-      "learning_rate": 7.829132092956586e-07,
-      "loss": 5.569941095018294e-07,
-      "num_tokens": 283603.0,
-      "reward": 0.6040733456611633,
-      "reward_std": 0.0834638923406601,
-      "rewards/true_env_reward_fn/mean": 0.6040733456611633,
-      "rewards/true_env_reward_fn/std": 0.08346389979124069,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 61.583335876464844,
+      "completions/mean_terminated_length": 61.583335876464844,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "entropy": 1.1755708158016205,
+      "epoch": 1.6829268292682928,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08657841384410858,
+      "kl": 2.0373249526528525e-05,
+      "learning_rate": 9.868477119388894e-07,
+      "loss": -0.01668858528137207,
+      "num_tokens": 1723155.0,
+      "reward": 0.39579567313194275,
+      "reward_std": 0.3625684082508087,
+      "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+      "rewards/true_env_reward_fn/std": 0.3625684380531311,
       "step": 69,
-      "step_time": 3.466609713001162
+      "step_time": 9.677535032999685
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 189.0,
-      "completions/max_terminated_length": 189.0,
-      "completions/mean_length": 126.0,
-      "completions/mean_terminated_length": 126.0,
-      "completions/min_length": 76.0,
-      "completions/min_terminated_length": 76.0,
-      "entropy": 1.8668264746665955,
-      "epoch": 0.5691056910569106,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.11100972443819046,
-      "kl": 1.3833193406753708e-05,
-      "learning_rate": 7.81801625108622e-07,
-      "loss": -0.04258224368095398,
-      "num_tokens": 290511.0,
-      "reward": 0.37345871329307556,
-      "reward_std": 0.016035744920372963,
-      "rewards/true_env_reward_fn/mean": 0.37345871329307556,
-      "rewards/true_env_reward_fn/std": 0.016035741195082664,
+      "completions/max_length": 212.0,
+      "completions/max_terminated_length": 212.0,
+      "completions/mean_length": 74.79167175292969,
+      "completions/mean_terminated_length": 74.79167175292969,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.296934336423874,
+      "epoch": 1.7073170731707317,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07736476510763168,
+      "kl": 2.0918997506669257e-05,
+      "learning_rate": 9.85860129488821e-07,
+      "loss": 0.03239107131958008,
+      "num_tokens": 1742689.0,
+      "reward": 0.6141302585601807,
+      "reward_std": 0.23138943314552307,
+      "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+      "rewards/true_env_reward_fn/std": 0.23138941824436188,
       "step": 70,
-      "step_time": 8.357124549000218
+      "step_time": 15.20990351499995
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 58.5,
-      "completions/mean_terminated_length": 58.5,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.1647167801856995,
-      "epoch": 0.5772357723577236,
+      "completions/max_length": 210.0,
+      "completions/max_terminated_length": 210.0,
+      "completions/mean_length": 66.125,
+      "completions/mean_terminated_length": 66.125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.2145576775074005,
+      "epoch": 1.7317073170731707,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12842044234275818,
-      "kl": 1.35402724481537e-05,
-      "learning_rate": 7.806558541484517e-07,
-      "loss": -0.0010651163756847382,
-      "num_tokens": 294315.0,
-      "reward": 0.6432806849479675,
-      "reward_std": 0.2300010770559311,
-      "rewards/true_env_reward_fn/mean": 0.6432806849479675,
-      "rewards/true_env_reward_fn/std": 0.23000109195709229,
+      "grad_norm": 0.07104668766260147,
+      "kl": 1.726100731502811e-05,
+      "learning_rate": 9.848373297716414e-07,
+      "loss": 0.03256790712475777,
+      "num_tokens": 1765463.0,
+      "reward": 0.48419874906539917,
+      "reward_std": 0.32040080428123474,
+      "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+      "rewards/true_env_reward_fn/std": 0.32040080428123474,
       "step": 71,
-      "step_time": 3.8402047919989855
+      "step_time": 14.703903473999844
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.1465299725532532,
-      "epoch": 0.5853658536585366,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23560228943824768,
-      "kl": 1.4576367902918719e-05,
-      "learning_rate": 7.794759990082466e-07,
-      "loss": -0.11232151836156845,
-      "num_tokens": 297803.0,
-      "reward": 0.30700522661209106,
-      "reward_std": 0.3690750300884247,
-      "rewards/true_env_reward_fn/mean": 0.30700522661209106,
-      "rewards/true_env_reward_fn/std": 0.3690750002861023,
+      "completions/max_length": 86.0,
+      "completions/max_terminated_length": 86.0,
+      "completions/mean_length": 62.35416793823242,
+      "completions/mean_terminated_length": 62.35416793823242,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.265857070684433,
+      "epoch": 1.7560975609756098,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07747533172369003,
+      "kl": 1.5618132920280914e-05,
+      "learning_rate": 9.837793869243467e-07,
+      "loss": -0.00018438976258039474,
+      "num_tokens": 1791512.0,
+      "reward": 0.45079630613327026,
+      "reward_std": 0.2226068526506424,
+      "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+      "rewards/true_env_reward_fn/std": 0.2226068526506424,
       "step": 72,
-      "step_time": 3.467162693001228
+      "step_time": 9.422353613000269
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 137.0,
-      "completions/max_terminated_length": 137.0,
-      "completions/mean_length": 73.5,
-      "completions/mean_terminated_length": 73.5,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "entropy": 1.2479569911956787,
-      "epoch": 0.5934959349593496,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011426099081290886,
-      "kl": 1.304310217165039e-05,
-      "learning_rate": 7.782621653330256e-07,
-      "loss": 6.391838383024151e-07,
-      "num_tokens": 301427.0,
-      "reward": 0.5610077381134033,
-      "reward_std": 0.316459059715271,
-      "rewards/true_env_reward_fn/mean": 0.5610077381134033,
-      "rewards/true_env_reward_fn/std": 0.3164590299129486,
+      "completions/max_length": 188.0,
+      "completions/max_terminated_length": 188.0,
+      "completions/mean_length": 73.0625,
+      "completions/mean_terminated_length": 73.0625,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.3557232320308685,
+      "epoch": 1.7804878048780488,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0811411589384079,
+      "kl": 1.9600361611082917e-05,
+      "learning_rate": 9.826863776312618e-07,
+      "loss": -0.019779374822974205,
+      "num_tokens": 1820731.0,
+      "reward": 0.431186318397522,
+      "reward_std": 0.23306044936180115,
+      "rewards/true_env_reward_fn/mean": 0.431186318397522,
+      "rewards/true_env_reward_fn/std": 0.23306044936180115,
       "step": 73,
-      "step_time": 5.824168748999
+      "step_time": 16.65922043799992
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 99.0,
-      "completions/max_terminated_length": 99.0,
-      "completions/mean_length": 63.125,
-      "completions/mean_terminated_length": 63.125,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.170280933380127,
-      "epoch": 0.6016260162601627,
+      "completions/max_length": 215.0,
+      "completions/max_terminated_length": 215.0,
+      "completions/mean_length": 78.375,
+      "completions/mean_terminated_length": 78.375,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2767037451267242,
+      "epoch": 1.8048780487804879,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22593456506729126,
-      "kl": 2.0052431864314713e-05,
-      "learning_rate": 7.77014461810269e-07,
-      "loss": 0.16111303865909576,
-      "num_tokens": 305492.0,
-      "reward": 0.3909183144569397,
-      "reward_std": 0.21756574511528015,
-      "rewards/true_env_reward_fn/mean": 0.3909183144569397,
-      "rewards/true_env_reward_fn/std": 0.21756574511528015,
+      "grad_norm": 0.08133924007415771,
+      "kl": 1.8058163732348476e-05,
+      "learning_rate": 9.815583811184808e-07,
+      "loss": -0.02447839081287384,
+      "num_tokens": 1841389.0,
+      "reward": 0.5825158953666687,
+      "reward_std": 0.2041907161474228,
+      "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+      "rewards/true_env_reward_fn/std": 0.20419073104858398,
       "step": 74,
-      "step_time": 4.510902927002462
+      "step_time": 13.422026366999944
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 63.75,
-      "completions/mean_terminated_length": 63.75,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2373355031013489,
-      "epoch": 0.6097560975609756,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.604005466215312e-05,
-      "kl": 1.0138399375136942e-05,
-      "learning_rate": 7.757330001601855e-07,
-      "loss": 5.069200028628984e-07,
-      "num_tokens": 309826.0,
-      "reward": 0.5905972719192505,
-      "reward_std": 0.15080371499061584,
-      "rewards/true_env_reward_fn/mean": 0.5905972719192505,
-      "rewards/true_env_reward_fn/std": 0.15080371499061584,
+      "completions/max_length": 90.0,
+      "completions/max_terminated_length": 90.0,
+      "completions/mean_length": 54.25,
+      "completions/mean_terminated_length": 54.25,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.2325710952281952,
+      "epoch": 1.8292682926829267,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.11014537513256073,
+      "kl": 3.268667387601454e-05,
+      "learning_rate": 9.803954791481238e-07,
+      "loss": 0.045359574258327484,
+      "num_tokens": 1871129.0,
+      "reward": 0.3935621678829193,
+      "reward_std": 0.22456605732440948,
+      "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+      "rewards/true_env_reward_fn/std": 0.2245660424232483,
       "step": 75,
-      "step_time": 3.6695911980004894
+      "step_time": 9.51117546200021
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 90.0,
-      "completions/max_terminated_length": 90.0,
-      "completions/mean_length": 66.875,
-      "completions/mean_terminated_length": 66.875,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.1873346865177155,
-      "epoch": 0.6178861788617886,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2067025899887085,
-      "kl": 1.4842833934380906e-05,
-      "learning_rate": 7.744178951257091e-07,
-      "loss": -0.036428727209568024,
-      "num_tokens": 316885.0,
-      "reward": 0.13499999046325684,
-      "reward_std": 0.23260429501533508,
-      "rewards/true_env_reward_fn/mean": 0.13499999046325684,
-      "rewards/true_env_reward_fn/std": 0.23260430991649628,
+      "completions/max_length": 134.0,
+      "completions/max_terminated_length": 134.0,
+      "completions/mean_length": 74.85417175292969,
+      "completions/mean_terminated_length": 74.85417175292969,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "entropy": 1.412343978881836,
+      "epoch": 1.8536585365853657,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07616850733757019,
+      "kl": 1.848336046350596e-05,
+      "learning_rate": 9.791977560124118e-07,
+      "loss": 0.030123719945549965,
+      "num_tokens": 1892706.0,
+      "reward": 0.5764689445495605,
+      "reward_std": 0.18864154815673828,
+      "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+      "rewards/true_env_reward_fn/std": 0.18864154815673828,
       "step": 76,
-      "step_time": 4.359561059001862
+      "step_time": 9.295928349999713
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0709484219551086,
-      "epoch": 0.6260162601626016,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18151503801345825,
-      "kl": 1.3910183042753488e-05,
-      "learning_rate": 7.730692644622251e-07,
-      "loss": -0.06179043650627136,
-      "num_tokens": 319230.0,
-      "reward": 0.6732838153839111,
-      "reward_std": 0.1450435221195221,
-      "rewards/true_env_reward_fn/mean": 0.6732838153839111,
-      "rewards/true_env_reward_fn/std": 0.14504355192184448,
+      "completions/max_length": 248.0,
+      "completions/max_terminated_length": 248.0,
+      "completions/mean_length": 71.54167175292969,
+      "completions/mean_terminated_length": 71.54167175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.342492938041687,
+      "epoch": 1.8780487804878048,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08141017705202103,
+      "kl": 1.587149881743244e-05,
+      "learning_rate": 9.779652985275562e-07,
+      "loss": -0.02852344512939453,
+      "num_tokens": 1915324.0,
+      "reward": 0.4926157593727112,
+      "reward_std": 0.20701222121715546,
+      "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+      "rewards/true_env_reward_fn/std": 0.20701222121715546,
       "step": 77,
-      "step_time": 3.1786108079995756
+      "step_time": 15.693113021000045
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 107.0,
-      "completions/max_terminated_length": 107.0,
-      "completions/mean_length": 72.375,
-      "completions/mean_terminated_length": 72.375,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.5439093112945557,
-      "epoch": 0.6341463414634146,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20670665800571442,
-      "kl": 1.7317805031780154e-05,
-      "learning_rate": 7.716872289270261e-07,
-      "loss": -0.0654018223285675,
-      "num_tokens": 324633.0,
-      "reward": 0.23838475346565247,
-      "reward_std": 0.2594907879829407,
-      "rewards/true_env_reward_fn/mean": 0.23838475346565247,
-      "rewards/true_env_reward_fn/std": 0.2594907879829407,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 256.0,
+      "completions/mean_length": 62.3125,
+      "completions/mean_terminated_length": 62.3125,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2102001011371613,
+      "epoch": 1.9024390243902438,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0555732287466526,
+      "kl": 1.6820984001242323e-05,
+      "learning_rate": 9.766981960274652e-07,
+      "loss": 0.041817761957645416,
+      "num_tokens": 1933867.0,
+      "reward": 0.5576165318489075,
+      "reward_std": 0.3197881579399109,
+      "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+      "rewards/true_env_reward_fn/std": 0.3197881281375885,
       "step": 78,
-      "step_time": 4.930556027000421
+      "step_time": 16.146651725000083
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 47.625,
-      "completions/mean_terminated_length": 47.625,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1996066868305206,
-      "epoch": 0.6422764227642277,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21137002110481262,
-      "kl": 1.325221819570288e-05,
-      "learning_rate": 7.702719122684991e-07,
-      "loss": 0.003889208659529686,
-      "num_tokens": 329142.0,
-      "reward": 0.3934500217437744,
-      "reward_std": 0.1389254629611969,
-      "rewards/true_env_reward_fn/mean": 0.3934500217437744,
-      "rewards/true_env_reward_fn/std": 0.1389254778623581,
+      "completions/max_length": 148.0,
+      "completions/max_terminated_length": 148.0,
+      "completions/mean_length": 69.25,
+      "completions/mean_terminated_length": 69.25,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.2909597754478455,
+      "epoch": 1.9268292682926829,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05769016966223717,
+      "kl": 1.534885905130068e-05,
+      "learning_rate": 9.753965403572702e-07,
+      "loss": -0.04179058223962784,
+      "num_tokens": 1953375.0,
+      "reward": 0.5617212057113647,
+      "reward_std": 0.18222570419311523,
+      "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+      "rewards/true_env_reward_fn/std": 0.18222568929195404,
       "step": 79,
-      "step_time": 3.5688320999997813
+      "step_time": 9.82867347299998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.125,
-      "completions/mean_terminated_length": 53.125,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.4094278812408447,
-      "epoch": 0.6504065040650406,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.17559278011322021,
-      "kl": 1.6261046312138205e-05,
-      "learning_rate": 7.688234412150453e-07,
-      "loss": -0.04887707903981209,
-      "num_tokens": 331663.0,
-      "reward": 0.49859046936035156,
-      "reward_std": 0.12171231955289841,
-      "rewards/true_env_reward_fn/mean": 0.49859046936035156,
-      "rewards/true_env_reward_fn/std": 0.12171231955289841,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 66.5,
+      "completions/mean_terminated_length": 66.5,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.2647078335285187,
+      "epoch": 1.951219512195122,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06353812664747238,
+      "kl": 2.195177648900426e-05,
+      "learning_rate": 9.740604258666668e-07,
+      "loss": -0.09541463106870651,
+      "num_tokens": 1978255.0,
+      "reward": 0.5184200406074524,
+      "reward_std": 0.28920501470565796,
+      "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+      "rewards/true_env_reward_fn/std": 0.28920501470565796,
       "step": 80,
-      "step_time": 3.7867210379990865
+      "step_time": 11.267316974000096
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 51.25,
-      "completions/mean_terminated_length": 51.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.1693094372749329,
-      "epoch": 0.6585365853658537,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010281114373356104,
-      "kl": 1.2930718639836414e-05,
-      "learning_rate": 7.673419454637328e-07,
-      "loss": 6.465359092544531e-07,
-      "num_tokens": 334637.0,
-      "reward": 0.5707399845123291,
-      "reward_std": 0.11909874528646469,
-      "rewards/true_env_reward_fn/mean": 0.5707399845123291,
-      "rewards/true_env_reward_fn/std": 0.1190987378358841,
+      "completions/max_length": 99.0,
+      "completions/max_terminated_length": 99.0,
+      "completions/mean_length": 62.72916793823242,
+      "completions/mean_terminated_length": 62.72916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3497782051563263,
+      "epoch": 1.975609756097561,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08975031226873398,
+      "kl": 3.0107988550298614e-05,
+      "learning_rate": 9.726899494030766e-07,
+      "loss": 0.04644065350294113,
+      "num_tokens": 2007634.0,
+      "reward": 0.3841831684112549,
+      "reward_std": 0.30559155344963074,
+      "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+      "rewards/true_env_reward_fn/std": 0.30559155344963074,
       "step": 81,
-      "step_time": 3.4751437539998733
+      "step_time": 10.035370067999793
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 59.5,
-      "completions/mean_terminated_length": 59.5,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "entropy": 1.3214005827903748,
-      "epoch": 0.6666666666666666,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 64.91667175292969,
+      "completions/mean_terminated_length": 64.91667175292969,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.2758312821388245,
+      "epoch": 2.0,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2361973226070404,
-      "kl": 1.4227861356630456e-05,
-      "learning_rate": 7.658275576686829e-07,
-      "loss": -0.08402466773986816,
-      "num_tokens": 341701.0,
-      "reward": 0.09331665933132172,
-      "reward_std": 0.2172754853963852,
-      "rewards/true_env_reward_fn/mean": 0.09331665933132172,
-      "rewards/true_env_reward_fn/std": 0.2172755002975464,
+      "grad_norm": 0.08747493475675583,
+      "kl": 1.806905720513896e-05,
+      "learning_rate": 9.71285210304628e-07,
+      "loss": -0.07698298245668411,
+      "num_tokens": 2024382.0,
+      "reward": 0.6160596609115601,
+      "reward_std": 0.23944181203842163,
+      "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+      "rewards/true_env_reward_fn/std": 0.23944182693958282,
       "step": 82,
-      "step_time": 4.433740980000948
+      "step_time": 9.56242024800008
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 512.0,
-      "completions/max_terminated_length": 126.0,
-      "completions/mean_length": 142.375,
-      "completions/mean_terminated_length": 89.5714340209961,
-      "completions/min_length": 62.0,
-      "completions/min_terminated_length": 62.0,
-      "entropy": 1.817092776298523,
-      "epoch": 0.6747967479674797,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.11815346032381058,
-      "kl": 1.6899173715501092e-05,
-      "learning_rate": 7.642804134291927e-07,
-      "loss": -0.09939523041248322,
-      "num_tokens": 346380.0,
-      "reward": 0.47429025173187256,
-      "reward_std": 0.24831563234329224,
-      "rewards/true_env_reward_fn/mean": 0.47429025173187256,
-      "rewards/true_env_reward_fn/std": 0.24831561744213104,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 114.0,
+      "completions/max_terminated_length": 114.0,
+      "completions/mean_length": 63.0625,
+      "completions/mean_terminated_length": 63.0625,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.2893573343753815,
+      "epoch": 2.024390243902439,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.05648891627788544,
+      "kl": 1.960936606337782e-05,
+      "learning_rate": 9.698463103929541e-07,
+      "loss": 0.05065512657165527,
+      "num_tokens": 2046817.0,
+      "reward": 0.5863184332847595,
+      "reward_std": 0.19063502550125122,
+      "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+      "rewards/true_env_reward_fn/std": 0.19063502550125122,
       "step": 83,
-      "step_time": 20.738665008999305
+      "step_time": 10.563381390999666
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 58.25,
-      "completions/mean_terminated_length": 58.25,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2211430668830872,
-      "epoch": 0.6829268292682927,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20952872931957245,
-      "kl": 1.2894654446427012e-05,
-      "learning_rate": 7.62700651277593e-07,
-      "loss": -0.0016747117042541504,
-      "num_tokens": 351186.0,
-      "reward": 0.386501669883728,
-      "reward_std": 0.17392057180404663,
-      "rewards/true_env_reward_fn/mean": 0.386501669883728,
-      "rewards/true_env_reward_fn/std": 0.17392057180404663,
+      "completions/max_length": 190.0,
+      "completions/max_terminated_length": 190.0,
+      "completions/mean_length": 75.29167175292969,
+      "completions/mean_terminated_length": 75.29167175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.2169642746448517,
+      "epoch": 2.048780487804878,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06511837989091873,
+      "kl": 2.2800771603215253e-05,
+      "learning_rate": 9.683733539658138e-07,
+      "loss": 0.02157626487314701,
+      "num_tokens": 2074535.0,
+      "reward": 0.4389227330684662,
+      "reward_std": 0.303769588470459,
+      "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+      "rewards/true_env_reward_fn/std": 0.303769588470459,
       "step": 84,
-      "step_time": 4.028964023000299
+      "step_time": 17.21621736799966
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 66.625,
-      "completions/mean_terminated_length": 66.625,
-      "completions/min_length": 53.0,
-      "completions/min_terminated_length": 53.0,
-      "entropy": 1.4367225170135498,
-      "epoch": 0.6910569105691057,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18939745426177979,
-      "kl": 1.6035403859859798e-05,
-      "learning_rate": 7.610884126668449e-07,
-      "loss": 0.0628451332449913,
-      "num_tokens": 355999.0,
-      "reward": 0.5092726349830627,
-      "reward_std": 0.2734805643558502,
-      "rewards/true_env_reward_fn/mean": 0.5092726349830627,
-      "rewards/true_env_reward_fn/std": 0.2734805941581726,
+      "completions/max_length": 432.0,
+      "completions/max_terminated_length": 432.0,
+      "completions/mean_length": 78.83333587646484,
+      "completions/mean_terminated_length": 78.83333587646484,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.396474927663803,
+      "epoch": 2.073170731707317,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.06777605414390564,
+      "kl": 2.369298363191774e-05,
+      "learning_rate": 9.66866447789531e-07,
+      "loss": -0.024554546922445297,
+      "num_tokens": 2096031.0,
+      "reward": 0.5134819746017456,
+      "reward_std": 0.28690314292907715,
+      "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+      "rewards/true_env_reward_fn/std": 0.28690314292907715,
       "step": 85,
-      "step_time": 4.244558566999331
+      "step_time": 25.519813745999954
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 52.0,
-      "completions/max_terminated_length": 52.0,
-      "completions/mean_length": 44.25,
-      "completions/mean_terminated_length": 44.25,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.009476900100708,
-      "epoch": 0.6991869918699187,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22703228890895844,
-      "kl": 1.2845549463236239e-05,
-      "learning_rate": 7.594438419578729e-07,
-      "loss": -0.005728684365749359,
-      "num_tokens": 360925.0,
-      "reward": 0.28028765320777893,
-      "reward_std": 0.2404259443283081,
-      "rewards/true_env_reward_fn/mean": 0.28028765320777893,
-      "rewards/true_env_reward_fn/std": 0.2404259443283081,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 65.95833587646484,
+      "completions/mean_terminated_length": 65.95833587646484,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.3048341274261475,
+      "epoch": 2.097560975609756,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07558907568454742,
+      "kl": 1.8465078937879298e-05,
+      "learning_rate": 9.653257010912558e-07,
+      "loss": -0.015101172029972076,
+      "num_tokens": 2122829.0,
+      "reward": 0.4031229019165039,
+      "reward_std": 0.22406692802906036,
+      "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+      "rewards/true_env_reward_fn/std": 0.22406692802906036,
       "step": 86,
-      "step_time": 2.618181756000922
+      "step_time": 10.78625990699993
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 57.25,
-      "completions/mean_terminated_length": 57.25,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.1686812043190002,
-      "epoch": 0.7073170731707317,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.401248098583892e-05,
-      "kl": 1.2304412848607171e-05,
-      "learning_rate": 7.577670864066391e-07,
-      "loss": 6.143833388705389e-07,
-      "num_tokens": 362399.0,
-      "reward": 0.768503725528717,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.768503725528717,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 178.0,
+      "completions/max_terminated_length": 178.0,
+      "completions/mean_length": 82.125,
+      "completions/mean_terminated_length": 82.125,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.3657839596271515,
+      "epoch": 2.1219512195121952,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07850468903779984,
+      "kl": 2.0332241774667636e-05,
+      "learning_rate": 9.637512255510474e-07,
+      "loss": 0.06651890277862549,
+      "num_tokens": 2151091.0,
+      "reward": 0.3940638303756714,
+      "reward_std": 0.2639860212802887,
+      "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+      "rewards/true_env_reward_fn/std": 0.2639860212802887,
       "step": 87,
-      "step_time": 3.34067542199773
+      "step_time": 13.604215705999877
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 71.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 59.125,
-      "completions/mean_terminated_length": 59.125,
-      "completions/min_length": 51.0,
-      "completions/min_terminated_length": 51.0,
-      "entropy": 1.0876938998699188,
-      "epoch": 0.7154471544715447,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010135328921023756,
-      "kl": 1.3493038295564475e-05,
-      "learning_rate": 7.560582961509586e-07,
-      "loss": 6.750068450855906e-07,
-      "num_tokens": 365500.0,
-      "reward": 0.6114685535430908,
-      "reward_std": 0.1678776890039444,
-      "rewards/true_env_reward_fn/mean": 0.6114685535430908,
-      "rewards/true_env_reward_fn/std": 0.1678776890039444,
+      "completions/clipped_ratio": 0.02083333395421505,
+      "completions/max_length": 512.0,
+      "completions/max_terminated_length": 84.0,
+      "completions/mean_length": 68.125,
+      "completions/mean_terminated_length": 58.680850982666016,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.0381308495998383,
+      "epoch": 2.1463414634146343,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06663572043180466,
+      "kl": 2.4382573428738397e-05,
+      "learning_rate": 9.621431352937787e-07,
+      "loss": -0.08434788882732391,
+      "num_tokens": 2177281.0,
+      "reward": 0.40229034423828125,
+      "reward_std": 0.3266920745372772,
+      "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+      "rewards/true_env_reward_fn/std": 0.3266920745372772,
       "step": 88,
-      "step_time": 3.3087227100004384
+      "step_time": 32.408574500999975
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 61.25,
-      "completions/mean_terminated_length": 61.25,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.0288619995117188,
-      "epoch": 0.7235772357723578,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00010261479474138469,
-      "kl": 1.3740621852775803e-05,
-      "learning_rate": 7.543176241970547e-07,
-      "loss": 6.875395683891838e-07,
-      "num_tokens": 369222.0,
-      "reward": 0.6557307243347168,
-      "reward_std": 0.2151959389448166,
-      "rewards/true_env_reward_fn/mean": 0.6557307243347168,
-      "rewards/true_env_reward_fn/std": 0.21519595384597778,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 60.8125,
+      "completions/mean_terminated_length": 60.8125,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.0944073796272278,
+      "epoch": 2.1707317073170733,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06779129058122635,
+      "kl": 2.3317856630455935e-05,
+      "learning_rate": 9.60501546880865e-07,
+      "loss": 0.019480882212519646,
+      "num_tokens": 2200208.0,
+      "reward": 0.5087729692459106,
+      "reward_std": 0.33071935176849365,
+      "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+      "rewards/true_env_reward_fn/std": 0.33071935176849365,
       "step": 89,
-      "step_time": 3.786183243999403
+      "step_time": 9.901715897000031
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 60.875,
-      "completions/mean_terminated_length": 60.875,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.1757304668426514,
-      "epoch": 0.7317073170731707,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2307090163230896,
-      "kl": 2.1445125639729667e-05,
-      "learning_rate": 7.525452264058595e-07,
-      "loss": 0.12042637169361115,
-      "num_tokens": 373465.0,
-      "reward": 0.4571714401245117,
-      "reward_std": 0.39374110102653503,
-      "rewards/true_env_reward_fn/mean": 0.4571714401245117,
-      "rewards/true_env_reward_fn/std": 0.39374107122421265,
+      "completions/max_length": 132.0,
+      "completions/max_terminated_length": 132.0,
+      "completions/mean_length": 65.4375,
+      "completions/mean_terminated_length": 65.4375,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.1683936715126038,
+      "epoch": 2.1951219512195124,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07440414279699326,
+      "kl": 1.8814549775925116e-05,
+      "learning_rate": 9.58826579301814e-07,
+      "loss": -0.03402159363031387,
+      "num_tokens": 2227789.0,
+      "reward": 0.40219685435295105,
+      "reward_std": 0.17851270735263824,
+      "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+      "rewards/true_env_reward_fn/std": 0.17851269245147705,
       "step": 90,
-      "step_time": 3.9787140030002774
+      "step_time": 11.152492722000034
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 71.0,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "entropy": 1.302090346813202,
-      "epoch": 0.7398373983739838,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.16624286770820618,
-      "kl": 1.6463789506815374e-05,
-      "learning_rate": 7.507412614790579e-07,
-      "loss": -0.05975423753261566,
-      "num_tokens": 378029.0,
-      "reward": 0.3388232886791229,
-      "reward_std": 0.2467346489429474,
-      "rewards/true_env_reward_fn/mean": 0.3388232886791229,
-      "rewards/true_env_reward_fn/std": 0.24673466384410858,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 68.22917175292969,
+      "completions/mean_terminated_length": 68.22917175292969,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.1628780961036682,
+      "epoch": 2.2195121951219514,
+      "frac_reward_zero_std": 0.8333333730697632,
+      "grad_norm": 0.03311198577284813,
+      "kl": 1.5094836498974473e-05,
+      "learning_rate": 9.57118353965601e-07,
+      "loss": 0.01087917946279049,
+      "num_tokens": 2252192.0,
+      "reward": 0.5357927083969116,
+      "reward_std": 0.18703003227710724,
+      "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+      "rewards/true_env_reward_fn/std": 0.18703003227710724,
       "step": 91,
-      "step_time": 3.9565000490001694
+      "step_time": 10.656350811000038
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 109.0,
-      "completions/max_terminated_length": 109.0,
-      "completions/mean_length": 77.75,
-      "completions/mean_terminated_length": 77.75,
-      "completions/min_length": 56.0,
-      "completions/min_terminated_length": 56.0,
-      "entropy": 1.2768036723136902,
-      "epoch": 0.7479674796747967,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10557293146848679,
-      "kl": 1.2602345123013947e-05,
-      "learning_rate": 7.489058909448776e-07,
-      "loss": -0.023296140134334564,
-      "num_tokens": 380883.0,
-      "reward": 0.5399107933044434,
-      "reward_std": 0.26432597637176514,
-      "rewards/true_env_reward_fn/mean": 0.5399107933044434,
-      "rewards/true_env_reward_fn/std": 0.26432597637176514,
+      "completions/max_length": 121.0,
+      "completions/max_terminated_length": 121.0,
+      "completions/mean_length": 66.97917175292969,
+      "completions/mean_terminated_length": 66.97917175292969,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "entropy": 1.2369268834590912,
+      "epoch": 2.2439024390243905,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08958107978105545,
+      "kl": 2.8437810669856844e-05,
+      "learning_rate": 9.553769946918698e-07,
+      "loss": 0.005673397332429886,
+      "num_tokens": 2274199.0,
+      "reward": 0.5484694242477417,
+      "reward_std": 0.27515000104904175,
+      "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+      "rewards/true_env_reward_fn/std": 0.27515000104904175,
       "step": 92,
-      "step_time": 4.720347813999979
+      "step_time": 10.304143018000104
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 63.0,
-      "completions/max_terminated_length": 63.0,
-      "completions/mean_length": 49.0,
-      "completions/mean_terminated_length": 49.0,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "entropy": 1.2670618891716003,
-      "epoch": 0.7560975609756098,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14662617444992065,
-      "kl": 1.812677373891347e-05,
-      "learning_rate": 7.470392791436244e-07,
-      "loss": -0.05785401538014412,
-      "num_tokens": 386095.0,
-      "reward": 0.30487915873527527,
-      "reward_std": 0.24597851932048798,
-      "rewards/true_env_reward_fn/mean": 0.30487915873527527,
-      "rewards/true_env_reward_fn/std": 0.24597853422164917,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 131.0,
+      "completions/mean_length": 69.0625,
+      "completions/mean_terminated_length": 69.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.374023586511612,
+      "epoch": 2.2682926829268295,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.03936443477869034,
+      "kl": 1.8785845441016136e-05,
+      "learning_rate": 9.53602627701956e-07,
+      "loss": -0.01821933500468731,
+      "num_tokens": 2302818.0,
+      "reward": 0.3851678967475891,
+      "reward_std": 0.2433396279811859,
+      "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+      "rewards/true_env_reward_fn/std": 0.2433396428823471,
       "step": 93,
-      "step_time": 3.1318131530006212
+      "step_time": 13.589426085000014
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 84.0,
-      "completions/max_terminated_length": 84.0,
-      "completions/mean_length": 54.5,
-      "completions/mean_terminated_length": 54.5,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.1208478510379791,
-      "epoch": 0.7642276422764228,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011506211740197614,
-      "kl": 1.2571507795655634e-05,
-      "learning_rate": 7.451415932129691e-07,
-      "loss": 6.294373520177032e-07,
-      "num_tokens": 388335.0,
-      "reward": 0.7244763970375061,
-      "reward_std": 0.23028412461280823,
-      "rewards/true_env_reward_fn/mean": 0.7244763970375061,
-      "rewards/true_env_reward_fn/std": 0.23028412461280823,
+      "completions/max_length": 216.0,
+      "completions/max_terminated_length": 216.0,
+      "completions/mean_length": 68.9375,
+      "completions/mean_terminated_length": 68.9375,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.3004788756370544,
+      "epoch": 2.292682926829268,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06094004213809967,
+      "kl": 1.9176507976226276e-05,
+      "learning_rate": 9.517953816097395e-07,
+      "loss": 0.023817429319024086,
+      "num_tokens": 2325071.0,
+      "reward": 0.6004310846328735,
+      "reward_std": 0.23291133344173431,
+      "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+      "rewards/true_env_reward_fn/std": 0.23291133344173431,
       "step": 94,
-      "step_time": 3.6959203189999243
+      "step_time": 14.587356482000132
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 62.25,
-      "completions/mean_terminated_length": 62.25,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1998488903045654,
-      "epoch": 0.7723577235772358,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12171207368373871,
-      "kl": 1.6534771020815242e-05,
-      "learning_rate": 7.432130030729804e-07,
-      "loss": 0.05708106979727745,
-      "num_tokens": 393029.0,
-      "reward": 0.29566600918769836,
-      "reward_std": 0.2818882167339325,
-      "rewards/true_env_reward_fn/mean": 0.29566600918769836,
-      "rewards/true_env_reward_fn/std": 0.2818882167339325,
+      "completions/max_length": 237.0,
+      "completions/max_terminated_length": 237.0,
+      "completions/mean_length": 66.47917175292969,
+      "completions/mean_terminated_length": 66.47917175292969,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.233375996351242,
+      "epoch": 2.317073170731707,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08650019764900208,
+      "kl": 1.89352349480032e-05,
+      "learning_rate": 9.499553874123212e-07,
+      "loss": 0.14434456825256348,
+      "num_tokens": 2347902.0,
+      "reward": 0.5542359948158264,
+      "reward_std": 0.18165862560272217,
+      "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+      "rewards/true_env_reward_fn/std": 0.18165862560272217,
       "step": 95,
-      "step_time": 4.322851452001487
+      "step_time": 14.689755582000089
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 49.375,
-      "completions/mean_terminated_length": 49.375,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.0649794340133667,
-      "epoch": 0.7804878048780488,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.2012680470943451,
-      "kl": 1.1399301456549438e-05,
-      "learning_rate": 7.412536814109106e-07,
-      "loss": -0.05478152632713318,
-      "num_tokens": 398112.0,
-      "reward": 0.23480799794197083,
-      "reward_std": 0.28209570050239563,
-      "rewards/true_env_reward_fn/mean": 0.23480799794197083,
-      "rewards/true_env_reward_fn/std": 0.282095730304718,
+      "completions/max_length": 173.0,
+      "completions/max_terminated_length": 173.0,
+      "completions/mean_length": 57.0625,
+      "completions/mean_terminated_length": 57.0625,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.224440723657608,
+      "epoch": 2.341463414634146,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07974361628293991,
+      "kl": 1.838593607317307e-05,
+      "learning_rate": 9.480827784805278e-07,
+      "loss": 0.03995979577302933,
+      "num_tokens": 2361401.0,
+      "reward": 0.6956334114074707,
+      "reward_std": 0.185209721326828,
+      "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+      "rewards/true_env_reward_fn/std": 0.185209721326828,
       "step": 96,
-      "step_time": 3.4046103930013487
+      "step_time": 10.379233056999965
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 56.5,
-      "completions/mean_terminated_length": 56.5,
-      "completions/min_length": 40.0,
-      "completions/min_terminated_length": 40.0,
-      "entropy": 1.4298859238624573,
-      "epoch": 0.7886178861788617,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2239074409008026,
-      "kl": 3.293174540885957e-05,
-      "learning_rate": 7.392638036657332e-07,
-      "loss": 0.09779056906700134,
-      "num_tokens": 402892.0,
-      "reward": 0.13796034455299377,
-      "reward_std": 0.22141560912132263,
-      "rewards/true_env_reward_fn/mean": 0.13796034455299377,
-      "rewards/true_env_reward_fn/std": 0.22141562402248383,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 60.25,
+      "completions/mean_terminated_length": 60.25,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.330334097146988,
+      "epoch": 2.3658536585365852,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08950946480035782,
+      "kl": 3.245086418246501e-05,
+      "learning_rate": 9.461776905492444e-07,
+      "loss": -0.03975849226117134,
+      "num_tokens": 2384437.0,
+      "reward": 0.49323582649230957,
+      "reward_std": 0.30376356840133667,
+      "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+      "rewards/true_env_reward_fn/std": 0.3037635385990143,
       "step": 97,
-      "step_time": 3.779275342998517
+      "step_time": 10.037491584999998
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 52.0,
-      "completions/mean_terminated_length": 52.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2070425152778625,
-      "epoch": 0.7967479674796748,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.19742031395435333,
-      "kl": 1.4374184502230491e-05,
-      "learning_rate": 7.372435480124337e-07,
-      "loss": -0.006231316365301609,
-      "num_tokens": 408052.0,
-      "reward": 0.43320000171661377,
-      "reward_std": 0.05237230286002159,
-      "rewards/true_env_reward_fn/mean": 0.43320000171661377,
-      "rewards/true_env_reward_fn/std": 0.052372295409440994,
+      "completions/max_length": 163.0,
+      "completions/max_terminated_length": 163.0,
+      "completions/mean_length": 63.8125,
+      "completions/mean_terminated_length": 63.8125,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2434260249137878,
+      "epoch": 2.3902439024390243,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09637262672185898,
+      "kl": 3.597719251047238e-05,
+      "learning_rate": 9.442402617075764e-07,
+      "loss": 0.008840052410960197,
+      "num_tokens": 2409676.0,
+      "reward": 0.47345292568206787,
+      "reward_std": 0.3432519733905792,
+      "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+      "rewards/true_env_reward_fn/std": 0.34325194358825684,
       "step": 98,
-      "step_time": 3.1304682769987267
+      "step_time": 13.073343929999965
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 106.0,
-      "completions/max_terminated_length": 106.0,
-      "completions/mean_length": 67.125,
-      "completions/mean_terminated_length": 67.125,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 0.9987849593162537,
-      "epoch": 0.8048780487804879,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.897383668227121e-05,
-      "kl": 1.2614300885616103e-05,
-      "learning_rate": 7.35193095346056e-07,
-      "loss": 6.314263600870618e-07,
-      "num_tokens": 409605.0,
-      "reward": 0.8541955947875977,
-      "reward_std": 0.09160846471786499,
-      "rewards/true_env_reward_fn/mean": 0.8541955947875977,
-      "rewards/true_env_reward_fn/std": 0.09160846471786499,
+      "completions/max_length": 101.0,
+      "completions/max_terminated_length": 101.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3253428936004639,
+      "epoch": 2.4146341463414633,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1034398227930069,
+      "kl": 3.889948129653931e-05,
+      "learning_rate": 9.422706323888396e-07,
+      "loss": 0.01636725291609764,
+      "num_tokens": 2433369.0,
+      "reward": 0.5016611218452454,
+      "reward_std": 0.3056275546550751,
+      "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+      "rewards/true_env_reward_fn/std": 0.3056274950504303,
       "step": 99,
-      "step_time": 4.13536422299876
+      "step_time": 9.465850557000067
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 86.0,
-      "completions/max_terminated_length": 86.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.0226224660873413,
-      "epoch": 0.8130081300813008,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.20178858935832977,
-      "kl": 1.0500047665118473e-05,
-      "learning_rate": 7.331126292655044e-07,
-      "loss": -0.17970919609069824,
-      "num_tokens": 411488.0,
-      "reward": 0.6963247060775757,
-      "reward_std": 0.18840119242668152,
-      "rewards/true_env_reward_fn/mean": 0.6963247060775757,
-      "rewards/true_env_reward_fn/std": 0.1884012222290039,
+      "completions/max_length": 121.0,
+      "completions/max_terminated_length": 121.0,
+      "completions/mean_length": 62.125,
+      "completions/mean_terminated_length": 62.125,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "entropy": 1.2484558820724487,
+      "epoch": 2.4390243902439024,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08669883012771606,
+      "kl": 1.584698543410923e-05,
+      "learning_rate": 9.402689453603814e-07,
+      "loss": 0.13139240443706512,
+      "num_tokens": 2458407.0,
+      "reward": 0.34693777561187744,
+      "reward_std": 0.35830602049827576,
+      "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+      "rewards/true_env_reward_fn/std": 0.35830605030059814,
       "step": 100,
-      "step_time": 3.7544156769981782
+      "step_time": 11.33050741000011
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.0,
-      "completions/max_terminated_length": 82.0,
-      "completions/mean_length": 59.0,
-      "completions/mean_terminated_length": 59.0,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2509461045265198,
-      "epoch": 0.8211382113821138,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22887632250785828,
-      "kl": 2.1612477212329395e-05,
-      "learning_rate": 7.310023360571047e-07,
-      "loss": 0.025605827569961548,
-      "num_tokens": 414080.0,
-      "reward": 0.588032603263855,
-      "reward_std": 0.11032751202583313,
-      "rewards/true_env_reward_fn/mean": 0.588032603263855,
-      "rewards/true_env_reward_fn/std": 0.11032749712467194,
+      "completions/max_length": 225.0,
+      "completions/max_terminated_length": 225.0,
+      "completions/mean_length": 68.77083587646484,
+      "completions/mean_terminated_length": 68.77083587646484,
+      "completions/min_length": 17.0,
+      "completions/min_terminated_length": 17.0,
+      "entropy": 1.2351897060871124,
+      "epoch": 2.4634146341463414,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06004978343844414,
+      "kl": 2.0037293097630027e-05,
+      "learning_rate": 9.382353457132317e-07,
+      "loss": -0.04131116345524788,
+      "num_tokens": 2483052.0,
+      "reward": 0.38015443086624146,
+      "reward_std": 0.34710174798965454,
+      "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+      "rewards/true_env_reward_fn/std": 0.34710174798965454,
       "step": 101,
-      "step_time": 3.625197022998691
+      "step_time": 16.478299477000064
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 176.0,
-      "completions/max_terminated_length": 176.0,
-      "completions/mean_length": 95.375,
-      "completions/mean_terminated_length": 95.375,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.5443179607391357,
-      "epoch": 0.8292682926829268,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.10614532232284546,
-      "kl": 1.492139062975184e-05,
-      "learning_rate": 7.28862404677924e-07,
-      "loss": 0.06531564146280289,
-      "num_tokens": 419835.0,
-      "reward": 0.07074306160211563,
-      "reward_std": 0.2918013632297516,
-      "rewards/true_env_reward_fn/mean": 0.07074306160211563,
-      "rewards/true_env_reward_fn/std": 0.2918013632297516,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 63.4375,
+      "completions/mean_terminated_length": 63.4375,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3605049848556519,
+      "epoch": 2.4878048780487805,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09010742604732513,
+      "kl": 3.810847783825011e-05,
+      "learning_rate": 9.361699808515875e-07,
+      "loss": 0.038716960698366165,
+      "num_tokens": 2510193.0,
+      "reward": 0.3458574712276459,
+      "reward_std": 0.30283215641975403,
+      "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+      "rewards/true_env_reward_fn/std": 0.30283215641975403,
       "step": 102,
-      "step_time": 7.796810614998321
+      "step_time": 11.344593008000174
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 101.0,
       "completions/max_terminated_length": 101.0,
-      "completions/mean_length": 55.25,
-      "completions/mean_terminated_length": 55.25,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.3223788738250732,
-      "epoch": 0.8373983739837398,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2421368807554245,
-      "kl": 3.265505938543356e-05,
-      "learning_rate": 7.266930267388503e-07,
-      "loss": -0.07752113044261932,
-      "num_tokens": 422773.0,
-      "reward": 0.33568501472473145,
-      "reward_std": 0.2780380845069885,
-      "rewards/true_env_reward_fn/mean": 0.33568501472473145,
-      "rewards/true_env_reward_fn/std": 0.2780380845069885,
+      "completions/mean_length": 65.22917175292969,
+      "completions/mean_terminated_length": 65.22917175292969,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 1.19815993309021,
+      "epoch": 2.5121951219512195,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07395386695861816,
+      "kl": 2.6301003344997298e-05,
+      "learning_rate": 9.340730004821265e-07,
+      "loss": 0.01458972692489624,
+      "num_tokens": 2529212.0,
+      "reward": 0.5586616396903992,
+      "reward_std": 0.20150764286518097,
+      "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+      "rewards/true_env_reward_fn/std": 0.20150764286518097,
       "step": 103,
-      "step_time": 4.313938073000827
+      "step_time": 8.135681302999728
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2899,27 +2899,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 93.0,
-      "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 67.25,
-      "completions/mean_terminated_length": 67.25,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.3332037329673767,
-      "epoch": 0.8455284552845529,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13883370161056519,
-      "kl": 2.1224042484391248e-05,
-      "learning_rate": 7.244943964874369e-07,
-      "loss": 0.021739646792411804,
-      "num_tokens": 426507.0,
-      "reward": 0.40595096349716187,
-      "reward_std": 0.2035457342863083,
-      "rewards/true_env_reward_fn/mean": 0.40595096349716187,
-      "rewards/true_env_reward_fn/std": 0.2035457193851471,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 166.0,
+      "completions/max_terminated_length": 166.0,
+      "completions/mean_length": 73.89583587646484,
+      "completions/mean_terminated_length": 73.89583587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2152214348316193,
+      "epoch": 2.5365853658536586,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08900879323482513,
+      "kl": 3.066915814997628e-05,
+      "learning_rate": 9.31944556603157e-07,
+      "loss": 0.08802390843629837,
+      "num_tokens": 2557007.0,
+      "reward": 0.4009184241294861,
+      "reward_std": 0.32733896374702454,
+      "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+      "rewards/true_env_reward_fn/std": 0.3273389935493469,
       "step": 104,
-      "step_time": 4.155937195999286
+      "step_time": 15.185034105999875
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 68.39583587646484,
+      "completions/mean_terminated_length": 68.39583587646484,
       "completions/min_length": 31.0,
       "completions/min_terminated_length": 31.0,
-      "entropy": 1.3391229510307312,
-      "epoch": 0.8536585365853658,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24205009639263153,
-      "kl": 2.5022183763212524e-05,
-      "learning_rate": 7.222667107905085e-07,
-      "loss": 0.06330433487892151,
-      "num_tokens": 429010.0,
-      "reward": 0.3355163037776947,
-      "reward_std": 0.2902730703353882,
-      "rewards/true_env_reward_fn/mean": 0.3355163037776947,
-      "rewards/true_env_reward_fn/std": 0.29027310013771057,
+      "entropy": 1.2849501073360443,
+      "epoch": 2.5609756097560976,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0703769251704216,
+      "kl": 1.9505746195136453e-05,
+      "learning_rate": 9.297848034936005e-07,
+      "loss": 0.036192238330841064,
+      "num_tokens": 2581170.0,
+      "reward": 0.4875798225402832,
+      "reward_std": 0.16742677986621857,
+      "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+      "rewards/true_env_reward_fn/std": 0.16742677986621857,
       "step": 105,
-      "step_time": 3.808478789000219
+      "step_time": 9.588520330999927
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 73.125,
-      "completions/mean_terminated_length": 73.125,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "entropy": 1.1864720582962036,
-      "epoch": 0.8617886178861789,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13473568856716156,
-      "kl": 1.4212585938366828e-05,
-      "learning_rate": 7.200101691165338e-07,
-      "loss": -0.020715661346912384,
-      "num_tokens": 432403.0,
-      "reward": 0.4871198534965515,
-      "reward_std": 0.15407639741897583,
-      "rewards/true_env_reward_fn/mean": 0.4871198534965515,
-      "rewards/true_env_reward_fn/std": 0.15407641232013702,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 66.95833587646484,
+      "completions/mean_terminated_length": 66.95833587646484,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.2687023878097534,
+      "epoch": 2.5853658536585367,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08327006548643112,
+      "kl": 2.3203040655062068e-05,
+      "learning_rate": 9.275938977018081e-07,
+      "loss": 0.003695126622915268,
+      "num_tokens": 2609408.0,
+      "reward": 0.40928101539611816,
+      "reward_std": 0.10633077472448349,
+      "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+      "rewards/true_env_reward_fn/std": 0.1063307598233223,
       "step": 106,
-      "step_time": 4.240638332001254
+      "step_time": 14.60399662399982
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 48.0,
-      "completions/mean_terminated_length": 48.0,
-      "completions/min_length": 24.0,
-      "completions/min_terminated_length": 24.0,
-      "entropy": 1.0669284462928772,
-      "epoch": 0.8699186991869918,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14131899178028107,
-      "kl": 1.5787159554747632e-05,
-      "learning_rate": 7.177249735177651e-07,
-      "loss": 0.03678784519433975,
-      "num_tokens": 435995.0,
-      "reward": 0.5010770559310913,
-      "reward_std": 0.48966261744499207,
-      "rewards/true_env_reward_fn/mean": 0.5010770559310913,
-      "rewards/true_env_reward_fn/std": 0.48966261744499207,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 79.79167175292969,
+      "completions/mean_terminated_length": 79.79167175292969,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "entropy": 1.1623006761074066,
+      "epoch": 2.6097560975609757,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.07009758800268173,
+      "kl": 2.6010310648416635e-05,
+      "learning_rate": 9.253719980342134e-07,
+      "loss": -0.025412028655409813,
+      "num_tokens": 2641446.0,
+      "reward": 0.29606160521507263,
+      "reward_std": 0.3615049123764038,
+      "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+      "rewards/true_env_reward_fn/std": 0.3615049123764038,
       "step": 107,
-      "step_time": 3.3587191269998584
+      "step_time": 20.100954443999854
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 91.0,
-      "completions/max_terminated_length": 91.0,
-      "completions/mean_length": 71.875,
-      "completions/mean_terminated_length": 71.875,
-      "completions/min_length": 57.0,
-      "completions/min_terminated_length": 57.0,
-      "entropy": 1.304731547832489,
-      "epoch": 0.8780487804878049,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.380985673284158e-05,
-      "kl": 1.3128728824085556e-05,
-      "learning_rate": 7.154113286121462e-07,
-      "loss": 6.494262834166875e-07,
-      "num_tokens": 442094.0,
-      "reward": 0.4055500030517578,
-      "reward_std": 0.052258480340242386,
-      "rewards/true_env_reward_fn/mean": 0.4055500030517578,
-      "rewards/true_env_reward_fn/std": 0.052258484065532684,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 60.6875,
+      "completions/mean_terminated_length": 60.6875,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
+      "entropy": 1.2318958044052124,
+      "epoch": 2.6341463414634148,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09370094537734985,
+      "kl": 2.169116805816884e-05,
+      "learning_rate": 9.23119265543822e-07,
+      "loss": -0.009763844311237335,
+      "num_tokens": 2659695.0,
+      "reward": 0.5309837460517883,
+      "reward_std": 0.1692933589220047,
+      "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+      "rewards/true_env_reward_fn/std": 0.1692933589220047,
       "step": 108,
-      "step_time": 4.337008413998774
+      "step_time": 8.304149297999857
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.2324069738388062,
-      "epoch": 0.8861788617886179,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13588950037956238,
-      "kl": 1.3448377558233915e-05,
-      "learning_rate": 7.130694415649912e-07,
-      "loss": 1.0952353477478027e-06,
-      "num_tokens": 447226.0,
-      "reward": 0.20854972302913666,
-      "reward_std": 0.06059705466032028,
-      "rewards/true_env_reward_fn/mean": 0.20854972302913666,
-      "rewards/true_env_reward_fn/std": 0.06059705838561058,
+      "completions/max_length": 117.0,
+      "completions/max_terminated_length": 117.0,
+      "completions/mean_length": 66.29167175292969,
+      "completions/mean_terminated_length": 66.29167175292969,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "entropy": 1.248624861240387,
+      "epoch": 2.658536585365854,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09214548021554947,
+      "kl": 5.0202284000988584e-05,
+      "learning_rate": 9.208358635185372e-07,
+      "loss": 0.0672653466463089,
+      "num_tokens": 2691005.0,
+      "reward": 0.306609183549881,
+      "reward_std": 0.24702024459838867,
+      "rewards/true_env_reward_fn/mean": 0.306609183549881,
+      "rewards/true_env_reward_fn/std": 0.24702024459838867,
       "step": 109,
-      "step_time": 3.2976038649994734
+      "step_time": 11.260021517000041
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 67.625,
-      "completions/mean_terminated_length": 67.625,
-      "completions/min_length": 55.0,
-      "completions/min_terminated_length": 55.0,
-      "entropy": 1.1567262411117554,
-      "epoch": 0.8943089430894309,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.736967720324174e-05,
-      "kl": 1.2838129805459175e-05,
-      "learning_rate": 7.106995220704342e-07,
-      "loss": 6.425898391171359e-07,
-      "num_tokens": 450359.0,
-      "reward": 0.7316612601280212,
-      "reward_std": 0.0,
-      "rewards/true_env_reward_fn/mean": 0.7316612601280212,
-      "rewards/true_env_reward_fn/std": 0.0,
+      "completions/max_length": 294.0,
+      "completions/max_terminated_length": 294.0,
+      "completions/mean_length": 77.64583587646484,
+      "completions/mean_terminated_length": 77.64583587646484,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 1.2180723249912262,
+      "epoch": 2.682926829268293,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08635839074850082,
+      "kl": 3.741631462617079e-05,
+      "learning_rate": 9.185219574693241e-07,
+      "loss": 0.06331576406955719,
+      "num_tokens": 2717196.0,
+      "reward": 0.5423221588134766,
+      "reward_std": 0.3347312808036804,
+      "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+      "rewards/true_env_reward_fn/std": 0.3347312808036804,
       "step": 110,
-      "step_time": 4.067084037998939
+      "step_time": 22.80178854000019
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 100.0,
       "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 65.0,
-      "completions/mean_terminated_length": 65.0,
-      "completions/min_length": 12.0,
-      "completions/min_terminated_length": 12.0,
-      "entropy": 1.496058464050293,
-      "epoch": 0.9024390243902439,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.18041981756687164,
-      "kl": 1.6616825632809196e-05,
-      "learning_rate": 7.083017823326532e-07,
-      "loss": 0.0269068144261837,
-      "num_tokens": 453583.0,
-      "reward": 0.5647265911102295,
-      "reward_std": 0.1507105529308319,
-      "rewards/true_env_reward_fn/mean": 0.5647265911102295,
-      "rewards/true_env_reward_fn/std": 0.1507105529308319,
+      "completions/mean_length": 55.5,
+      "completions/mean_terminated_length": 55.5,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2095272839069366,
+      "epoch": 2.7073170731707314,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09482823312282562,
+      "kl": 3.445757738518296e-05,
+      "learning_rate": 9.161777151182135e-07,
+      "loss": -0.007809684611856937,
+      "num_tokens": 2739924.0,
+      "reward": 0.4738404154777527,
+      "reward_std": 0.2762244939804077,
+      "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+      "rewards/true_env_reward_fn/std": 0.2762244939804077,
       "step": 111,
-      "step_time": 4.347732382997492
+      "step_time": 9.663163859000178
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 108.0,
-      "completions/max_terminated_length": 108.0,
-      "completions/mean_length": 67.75,
-      "completions/mean_terminated_length": 67.75,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.4196155667304993,
-      "epoch": 0.9105691056910569,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18451112508773804,
-      "kl": 2.1803500203532167e-05,
-      "learning_rate": 7.058764370468698e-07,
-      "loss": 0.1650262475013733,
-      "num_tokens": 456773.0,
-      "reward": 0.6907394528388977,
-      "reward_std": 0.1393815129995346,
-      "rewards/true_env_reward_fn/mean": 0.6907394528388977,
-      "rewards/true_env_reward_fn/std": 0.1393815129995346,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 72.79167175292969,
+      "completions/mean_terminated_length": 72.79167175292969,
+      "completions/min_length": 38.0,
+      "completions/min_terminated_length": 38.0,
+      "entropy": 1.339354246854782,
+      "epoch": 2.7317073170731705,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09729615598917007,
+      "kl": 4.7237631861207774e-05,
+      "learning_rate": 9.138033063861434e-07,
+      "loss": 0.0440620519220829,
+      "num_tokens": 2763226.0,
+      "reward": 0.4624016284942627,
+      "reward_std": 0.2299472838640213,
+      "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+      "rewards/true_env_reward_fn/std": 0.2299472540616989,
       "step": 112,
-      "step_time": 4.627644968999448
+      "step_time": 9.903081222999617
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 80.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 61.125,
-      "completions/mean_terminated_length": 61.125,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
-      "entropy": 1.2243221998214722,
-      "epoch": 0.9186991869918699,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13331371545791626,
-      "kl": 1.548633599668392e-05,
-      "learning_rate": 7.034237033801247e-07,
-      "loss": 0.039844345301389694,
-      "num_tokens": 462714.0,
-      "reward": 0.21676866710186005,
-      "reward_std": 0.26559779047966003,
-      "rewards/true_env_reward_fn/mean": 0.21676866710186005,
-      "rewards/true_env_reward_fn/std": 0.26559779047966003,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 62.5625,
+      "completions/mean_terminated_length": 62.5625,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.2139239311218262,
+      "epoch": 2.7560975609756095,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.09817806631326675,
+      "kl": 5.5064369917090517e-05,
+      "learning_rate": 9.113989033806433e-07,
+      "loss": 0.03889988735318184,
+      "num_tokens": 2788677.0,
+      "reward": 0.3767920434474945,
+      "reward_std": 0.3360261619091034,
+      "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+      "rewards/true_env_reward_fn/std": 0.3360261619091034,
       "step": 113,
-      "step_time": 3.8455466220002563
+      "step_time": 10.101770388999284
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 73.0,
-      "completions/mean_terminated_length": 73.0,
-      "completions/min_length": 58.0,
-      "completions/min_terminated_length": 58.0,
-      "entropy": 1.325823724269867,
-      "epoch": 0.926829268292683,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1866220384836197,
-      "kl": 1.8801019905367866e-05,
-      "learning_rate": 7.009438009518325e-07,
-      "loss": 0.06504581868648529,
-      "num_tokens": 465994.0,
-      "reward": 0.5194582939147949,
-      "reward_std": 0.2796703577041626,
-      "rewards/true_env_reward_fn/mean": 0.5194582939147949,
-      "rewards/true_env_reward_fn/std": 0.2796703577041626,
+      "completions/max_length": 236.0,
+      "completions/max_terminated_length": 236.0,
+      "completions/mean_length": 69.29167175292969,
+      "completions/mean_terminated_length": 69.29167175292969,
+      "completions/min_length": 25.0,
+      "completions/min_terminated_length": 25.0,
+      "entropy": 1.2278488278388977,
+      "epoch": 2.7804878048780486,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09011705964803696,
+      "kl": 3.285038519607042e-05,
+      "learning_rate": 9.089646803833588e-07,
+      "loss": 0.03598163276910782,
+      "num_tokens": 2812139.0,
+      "reward": 0.5151569843292236,
+      "reward_std": 0.24896851181983948,
+      "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+      "rewards/true_env_reward_fn/std": 0.24896851181983948,
       "step": 114,
-      "step_time": 4.151028698999653
+      "step_time": 17.633509853000305
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 57.625,
-      "completions/mean_terminated_length": 57.625,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.1521879434585571,
-      "epoch": 0.9349593495934959,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14320029318332672,
-      "kl": 1.2749982033710694e-05,
-      "learning_rate": 6.98436951814117e-07,
-      "loss": 0.03685300797224045,
-      "num_tokens": 468615.0,
-      "reward": 0.5399107933044434,
-      "reward_std": 0.26432597637176514,
-      "rewards/true_env_reward_fn/mean": 0.5399107933044434,
-      "rewards/true_env_reward_fn/std": 0.26432597637176514,
+      "completions/max_length": 151.0,
+      "completions/max_terminated_length": 151.0,
+      "completions/mean_length": 61.97916793823242,
+      "completions/mean_terminated_length": 61.97916793823242,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.1842038929462433,
+      "epoch": 2.8048780487804876,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07707802206277847,
+      "kl": 4.165519931120798e-05,
+      "learning_rate": 9.065008138374188e-07,
+      "loss": 0.03350803256034851,
+      "num_tokens": 2835354.0,
+      "reward": 0.4122808873653412,
+      "reward_std": 0.27231934666633606,
+      "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+      "rewards/true_env_reward_fn/std": 0.27231931686401367,
       "step": 115,
-      "step_time": 3.6973990600017714
+      "step_time": 12.307247350000125
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 56.0,
-      "completions/mean_terminated_length": 56.0,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.201507806777954,
-      "epoch": 0.943089430894309,
+      "completions/max_length": 232.0,
+      "completions/max_terminated_length": 232.0,
+      "completions/mean_length": 83.64583587646484,
+      "completions/mean_terminated_length": 83.64583587646484,
+      "completions/min_length": 37.0,
+      "completions/min_terminated_length": 37.0,
+      "entropy": 1.3357974886894226,
+      "epoch": 2.8292682926829267,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14264807105064392,
-      "kl": 2.6679515940486453e-05,
-      "learning_rate": 6.959033804319283e-07,
-      "loss": -0.023484818637371063,
-      "num_tokens": 471647.0,
-      "reward": 0.41836902499198914,
-      "reward_std": 0.3116860091686249,
-      "rewards/true_env_reward_fn/mean": 0.41836902499198914,
-      "rewards/true_env_reward_fn/std": 0.3116860091686249,
+      "grad_norm": 0.06329861283302307,
+      "kl": 1.8487026636648807e-05,
+      "learning_rate": 9.040074823346464e-07,
+      "loss": 0.030132077634334564,
+      "num_tokens": 2859017.0,
+      "reward": 0.5723411440849304,
+      "reward_std": 0.21183526515960693,
+      "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+      "rewards/true_env_reward_fn/std": 0.21183528006076813,
       "step": 116,
-      "step_time": 3.1295652919998247
+      "step_time": 14.468690254000194
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 62.375,
-      "completions/mean_terminated_length": 62.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.2834057807922363,
-      "epoch": 0.9512195121951219,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.810227154754102e-05,
-      "kl": 1.4841665233689127e-05,
-      "learning_rate": 6.933433136629443e-07,
-      "loss": 7.425555850204546e-07,
-      "num_tokens": 474682.0,
-      "reward": 0.6203632950782776,
-      "reward_std": 0.11898252367973328,
-      "rewards/true_env_reward_fn/mean": 0.6203632950782776,
-      "rewards/true_env_reward_fn/std": 0.11898253113031387,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 70.35417175292969,
+      "completions/mean_terminated_length": 70.35417175292969,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 1.1871840357780457,
+      "epoch": 2.8536585365853657,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06702237576246262,
+      "kl": 2.817388576659141e-05,
+      "learning_rate": 9.014848666026138e-07,
+      "loss": 0.00200769305229187,
+      "num_tokens": 2889050.0,
+      "reward": 0.3500348925590515,
+      "reward_std": 0.30559059977531433,
+      "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+      "rewards/true_env_reward_fn/std": 0.3055906295776367,
       "step": 117,
-      "step_time": 3.4368692790012574
+      "step_time": 11.849063975999798
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 63.625,
-      "completions/mean_terminated_length": 63.625,
-      "completions/min_length": 42.0,
-      "completions/min_terminated_length": 42.0,
-      "entropy": 1.191932499408722,
-      "epoch": 0.959349593495935,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.21204856038093567,
-      "kl": 3.64198385796044e-05,
-      "learning_rate": 6.907569807372574e-07,
-      "loss": -0.001312553882598877,
-      "num_tokens": 477027.0,
-      "reward": 0.5300568342208862,
-      "reward_std": 0.2945883274078369,
-      "rewards/true_env_reward_fn/mean": 0.5300568342208862,
-      "rewards/true_env_reward_fn/std": 0.2945883274078369,
+      "completions/max_length": 239.0,
+      "completions/max_terminated_length": 239.0,
+      "completions/mean_length": 79.4375,
+      "completions/mean_terminated_length": 79.4375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2978005111217499,
+      "epoch": 2.8780487804878048,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.0668371245265007,
+      "kl": 3.733048197318567e-05,
+      "learning_rate": 8.989331494915416e-07,
+      "loss": -0.04680684953927994,
+      "num_tokens": 2917335.0,
+      "reward": 0.32068905234336853,
+      "reward_std": 0.30586519837379456,
+      "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+      "rewards/true_env_reward_fn/std": 0.30586519837379456,
       "step": 118,
-      "step_time": 3.8569856240010267
+      "step_time": 16.597334930000216
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 190.0,
-      "completions/max_terminated_length": 190.0,
-      "completions/mean_length": 96.5,
-      "completions/mean_terminated_length": 96.5,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 1.2401175498962402,
-      "epoch": 0.967479674796748,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00011377666669432074,
-      "kl": 1.3742283954343293e-05,
-      "learning_rate": 6.881446132368494e-07,
-      "loss": 6.866695230201003e-07,
-      "num_tokens": 481999.0,
-      "reward": 0.5930472612380981,
-      "reward_std": 0.14818456768989563,
-      "rewards/true_env_reward_fn/mean": 0.5930472612380981,
-      "rewards/true_env_reward_fn/std": 0.14818456768989563,
+      "completions/max_length": 164.0,
+      "completions/max_terminated_length": 164.0,
+      "completions/mean_length": 69.10417175292969,
+      "completions/mean_terminated_length": 69.10417175292969,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2123413980007172,
+      "epoch": 2.902439024390244,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.074281245470047,
+      "kl": 3.654057309177006e-05,
+      "learning_rate": 8.963525159610464e-07,
+      "loss": 0.0373641662299633,
+      "num_tokens": 2938004.0,
+      "reward": 0.556015133857727,
+      "reward_std": 0.22209766507148743,
+      "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+      "rewards/true_env_reward_fn/std": 0.22209767997264862,
       "step": 119,
-      "step_time": 8.09440958399864
+      "step_time": 11.729475523999554
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 66.125,
-      "completions/mean_terminated_length": 66.125,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.230682611465454,
-      "epoch": 0.975609756097561,
+      "completions/max_length": 135.0,
+      "completions/max_terminated_length": 135.0,
+      "completions/mean_length": 66.08333587646484,
+      "completions/mean_terminated_length": 66.08333587646484,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.3360244035720825,
+      "epoch": 2.926829268292683,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.22175048291683197,
-      "kl": 1.2522132237791084e-05,
-      "learning_rate": 6.855064450748555e-07,
-      "loss": -0.04083740711212158,
-      "num_tokens": 490884.0,
-      "reward": 0.13476666808128357,
-      "reward_std": 0.2987530529499054,
-      "rewards/true_env_reward_fn/mean": 0.13476666808128357,
-      "rewards/true_env_reward_fn/std": 0.2987530827522278,
+      "grad_norm": 0.09382818639278412,
+      "kl": 3.540705620252993e-05,
+      "learning_rate": 8.937431530667327e-07,
+      "loss": 0.057918041944503784,
+      "num_tokens": 2966976.0,
+      "reward": 0.3999954164028168,
+      "reward_std": 0.2351321578025818,
+      "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+      "rewards/true_env_reward_fn/std": 0.23513217270374298,
       "step": 120,
-      "step_time": 4.678523641001448
+      "step_time": 11.503627788000358
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.0,
-      "completions/max_terminated_length": 81.0,
-      "completions/mean_length": 63.5,
-      "completions/mean_terminated_length": 63.5,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.2799639105796814,
-      "epoch": 0.983739837398374,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.19926966726779938,
-      "kl": 1.7022688552970067e-05,
-      "learning_rate": 6.828427124746189e-07,
-      "loss": -0.010804429650306702,
-      "num_tokens": 496404.0,
-      "reward": 0.24633333086967468,
-      "reward_std": 0.2454334795475006,
-      "rewards/true_env_reward_fn/mean": 0.24633333086967468,
-      "rewards/true_env_reward_fn/std": 0.2454334795475006,
+      "completions/max_length": 98.0,
+      "completions/max_terminated_length": 98.0,
+      "completions/mean_length": 58.97916793823242,
+      "completions/mean_terminated_length": 58.97916793823242,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2053601145744324,
+      "epoch": 2.951219512195122,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07024823874235153,
+      "kl": 3.6033439755556174e-05,
+      "learning_rate": 8.911052499466356e-07,
+      "loss": 0.04910843074321747,
+      "num_tokens": 2987391.0,
+      "reward": 0.5365906953811646,
+      "reward_std": 0.19872017204761505,
+      "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+      "rewards/true_env_reward_fn/std": 0.19872015714645386,
       "step": 121,
-      "step_time": 3.98071062300005
+      "step_time": 8.728293746000418
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 73.0,
-      "completions/max_terminated_length": 73.0,
-      "completions/mean_length": 46.875,
-      "completions/mean_terminated_length": 46.875,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.3840235471725464,
-      "epoch": 0.991869918699187,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.24853822588920593,
-      "kl": 3.688259130285587e-05,
-      "learning_rate": 6.801536539485403e-07,
-      "loss": 0.10205884277820587,
-      "num_tokens": 499767.0,
-      "reward": 0.3045905530452728,
-      "reward_std": 0.262839138507843,
-      "rewards/true_env_reward_fn/mean": 0.3045905530452728,
-      "rewards/true_env_reward_fn/std": 0.262839138507843,
+      "completions/max_length": 219.0,
+      "completions/max_terminated_length": 219.0,
+      "completions/mean_length": 70.8125,
+      "completions/mean_terminated_length": 70.8125,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "entropy": 1.296659678220749,
+      "epoch": 2.975609756097561,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0607762485742569,
+      "kl": 3.175417168677086e-05,
+      "learning_rate": 8.884389978075097e-07,
+      "loss": -0.040265124291181564,
+      "num_tokens": 3009358.0,
+      "reward": 0.49613699316978455,
+      "reward_std": 0.2080756276845932,
+      "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+      "rewards/true_env_reward_fn/std": 0.2080756276845932,
       "step": 122,
-      "step_time": 3.3792565210005705
+      "step_time": 15.51957702100026
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 50.375,
-      "completions/mean_terminated_length": 50.375,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 1.2064164280891418,
-      "epoch": 1.0,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1365528553724289,
-      "kl": 2.434901080050622e-05,
-      "learning_rate": 6.774395102767203e-07,
-      "loss": -0.03472680225968361,
-      "num_tokens": 504906.0,
-      "reward": 0.2722649872303009,
-      "reward_std": 0.2922348082065582,
-      "rewards/true_env_reward_fn/mean": 0.2722649872303009,
-      "rewards/true_env_reward_fn/std": 0.2922348082065582,
+      "completions/max_length": 93.0,
+      "completions/max_terminated_length": 93.0,
+      "completions/mean_length": 59.4375,
+      "completions/mean_terminated_length": 59.4375,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.2992768585681915,
+      "epoch": 3.0,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.09044087678194046,
+      "kl": 6.319149179034866e-05,
+      "learning_rate": 8.857445899109715e-07,
+      "loss": -0.030733143910765648,
+      "num_tokens": 3035563.0,
+      "reward": 0.34821078181266785,
+      "reward_std": 0.2354777753353119,
+      "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+      "rewards/true_env_reward_fn/std": 0.23547779023647308,
       "step": 123,
-      "step_time": 3.0233660449994204
+      "step_time": 8.471463828000196
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.2819936871528625,
-      "epoch": 1.008130081300813,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00023045104171615094,
-      "kl": 2.2608143808611203e-05,
-      "learning_rate": 6.747005244854004e-07,
-      "loss": 1.1284330412308918e-06,
-      "num_tokens": 508329.0,
-      "reward": 0.3149532079696655,
-      "reward_std": 0.4275679290294647,
-      "rewards/true_env_reward_fn/mean": 0.3149532079696655,
-      "rewards/true_env_reward_fn/std": 0.4275679886341095,
+      "completions/max_length": 193.0,
+      "completions/max_terminated_length": 193.0,
+      "completions/mean_length": 69.64583587646484,
+      "completions/mean_terminated_length": 69.64583587646484,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "entropy": 1.2003771364688873,
+      "epoch": 3.024390243902439,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08278124779462814,
+      "kl": 2.7146410047862446e-05,
+      "learning_rate": 8.83022221559489e-07,
+      "loss": 0.02903313934803009,
+      "num_tokens": 3056090.0,
+      "reward": 0.5313810110092163,
+      "reward_std": 0.18604923784732819,
+      "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+      "rewards/true_env_reward_fn/std": 0.18604923784732819,
       "step": 124,
-      "step_time": 4.01701365199915
+      "step_time": 13.438758649999727
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 157.0,
-      "completions/max_terminated_length": 157.0,
-      "completions/mean_length": 80.875,
-      "completions/mean_terminated_length": 80.875,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "entropy": 1.1542360186576843,
-      "epoch": 1.016260162601626,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.232049003709108e-05,
-      "kl": 1.291002809011843e-05,
-      "learning_rate": 6.719369418252023e-07,
-      "loss": 6.488799613180163e-07,
-      "num_tokens": 515076.0,
-      "reward": 0.4841846525669098,
-      "reward_std": 0.12780573964118958,
-      "rewards/true_env_reward_fn/mean": 0.4841846525669098,
-      "rewards/true_env_reward_fn/std": 0.12780575454235077,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 63.625,
+      "completions/mean_terminated_length": 63.625,
+      "completions/min_length": 41.0,
+      "completions/min_terminated_length": 41.0,
+      "entropy": 1.2338614165782928,
+      "epoch": 3.048780487804878,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.06770245730876923,
+      "kl": 2.570231345089269e-05,
+      "learning_rate": 8.802720900822269e-07,
+      "loss": 0.0324365571141243,
+      "num_tokens": 3080424.0,
+      "reward": 0.44920405745506287,
+      "reward_std": 0.206027552485466,
+      "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+      "rewards/true_env_reward_fn/std": 0.2060275673866272,
       "step": 125,
-      "step_time": 7.240956699999515
+      "step_time": 8.654177170999901
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 56.375,
-      "completions/mean_terminated_length": 56.375,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.2594389915466309,
-      "epoch": 1.024390243902439,
+      "completions/max_length": 129.0,
+      "completions/max_terminated_length": 129.0,
+      "completions/mean_length": 70.10417175292969,
+      "completions/mean_terminated_length": 70.10417175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.1446799635887146,
+      "epoch": 3.073170731707317,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13091468811035156,
-      "kl": 1.751603304001037e-05,
-      "learning_rate": 6.691490097491675e-07,
-      "loss": -0.033413223922252655,
-      "num_tokens": 520279.0,
-      "reward": 0.28095200657844543,
-      "reward_std": 0.21837711334228516,
-      "rewards/true_env_reward_fn/mean": 0.28095200657844543,
-      "rewards/true_env_reward_fn/std": 0.21837712824344635,
+      "grad_norm": 0.060568179935216904,
+      "kl": 3.4001183394138934e-05,
+      "learning_rate": 8.774943948207425e-07,
+      "loss": -0.009533079341053963,
+      "num_tokens": 3100469.0,
+      "reward": 0.5536229610443115,
+      "reward_std": 0.29822590947151184,
+      "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+      "rewards/true_env_reward_fn/std": 0.29822590947151184,
       "step": 126,
-      "step_time": 3.355879656997786
+      "step_time": 10.513378469000145
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 136.0,
-      "completions/max_terminated_length": 136.0,
-      "completions/mean_length": 77.875,
-      "completions/mean_terminated_length": 77.875,
-      "completions/min_length": 43.0,
-      "completions/min_terminated_length": 43.0,
-      "entropy": 1.3044686317443848,
-      "epoch": 1.032520325203252,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12953205406665802,
-      "kl": 1.8700401597016025e-05,
-      "learning_rate": 6.663369778906008e-07,
-      "loss": 0.03562816232442856,
-      "num_tokens": 524582.0,
-      "reward": 0.4330660402774811,
-      "reward_std": 0.4592672288417816,
-      "rewards/true_env_reward_fn/mean": 0.4330660402774811,
-      "rewards/true_env_reward_fn/std": 0.459267258644104,
+      "completions/max_length": 348.0,
+      "completions/max_terminated_length": 348.0,
+      "completions/mean_length": 75.5,
+      "completions/mean_terminated_length": 75.5,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3296749591827393,
+      "epoch": 3.097560975609756,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06767670065164566,
+      "kl": 4.0856727537175175e-05,
+      "learning_rate": 8.746893371145365e-07,
+      "loss": -0.023851895704865456,
+      "num_tokens": 3127557.0,
+      "reward": 0.3543795943260193,
+      "reward_std": 0.3506966233253479,
+      "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+      "rewards/true_env_reward_fn/std": 0.3506965935230255,
       "step": 127,
-      "step_time": 5.965807722999671
+      "step_time": 23.20779430600078
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 117.0,
-      "completions/max_terminated_length": 117.0,
-      "completions/mean_length": 75.375,
-      "completions/mean_terminated_length": 75.375,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "entropy": 1.1742327809333801,
-      "epoch": 1.040650406504065,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14993594586849213,
-      "kl": 1.1459212601039326e-05,
-      "learning_rate": 6.635010980407174e-07,
-      "loss": 0.03646668791770935,
-      "num_tokens": 526213.0,
-      "reward": 0.7185037136077881,
-      "reward_std": 0.1414213478565216,
-      "rewards/true_env_reward_fn/mean": 0.7185037136077881,
-      "rewards/true_env_reward_fn/std": 0.1414213627576828,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 62.520835876464844,
+      "completions/mean_terminated_length": 62.520835876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.369004338979721,
+      "epoch": 3.1219512195121952,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.08670635521411896,
+      "kl": 3.454186935414327e-05,
+      "learning_rate": 8.718571202864597e-07,
+      "loss": 0.03175315260887146,
+      "num_tokens": 3154478.0,
+      "reward": 0.37598031759262085,
+      "reward_std": 0.32647329568862915,
+      "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+      "rewards/true_env_reward_fn/std": 0.32647326588630676,
       "step": 128,
-      "step_time": 4.9305356690001645
+      "step_time": 11.551069149999876
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.0,
-      "completions/max_terminated_length": 64.0,
-      "completions/mean_length": 49.125,
-      "completions/mean_terminated_length": 49.125,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.0784690976142883,
-      "epoch": 1.048780487804878,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16007214784622192,
-      "kl": 1.2491957932070363e-05,
-      "learning_rate": 6.606416241260979e-07,
-      "loss": 0.006608985364437103,
-      "num_tokens": 531862.0,
-      "reward": 0.2934249937534332,
-      "reward_std": 0.2395382523536682,
-      "rewards/true_env_reward_fn/mean": 0.2934249937534332,
-      "rewards/true_env_reward_fn/std": 0.23953823745250702,
+      "completions/max_length": 113.0,
+      "completions/max_terminated_length": 113.0,
+      "completions/mean_length": 68.27083587646484,
+      "completions/mean_terminated_length": 68.27083587646484,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2507834732532501,
+      "epoch": 3.1463414634146343,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.07138162106275558,
+      "kl": 3.6777758396056015e-05,
+      "learning_rate": 8.689979496279746e-07,
+      "loss": 0.001895703375339508,
+      "num_tokens": 3182339.0,
+      "reward": 0.3563499450683594,
+      "reward_std": 0.2783089876174927,
+      "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+      "rewards/true_env_reward_fn/std": 0.2783089876174927,
       "step": 129,
-      "step_time": 3.173622508000335
+      "step_time": 10.723005456000465
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 65.875,
-      "completions/mean_terminated_length": 65.875,
-      "completions/min_length": 39.0,
-      "completions/min_terminated_length": 39.0,
-      "entropy": 1.308219850063324,
-      "epoch": 1.056910569105691,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.985446427483112e-05,
-      "kl": 1.2420873190421844e-05,
-      "learning_rate": 6.577588121859508e-07,
-      "loss": 6.241918413252279e-07,
-      "num_tokens": 535957.0,
-      "reward": 0.4817493259906769,
-      "reward_std": 0.029202036559581757,
-      "rewards/true_env_reward_fn/mean": 0.4817493259906769,
-      "rewards/true_env_reward_fn/std": 0.029202038422226906,
+      "completions/max_length": 160.0,
+      "completions/max_terminated_length": 160.0,
+      "completions/mean_length": 71.3125,
+      "completions/mean_terminated_length": 71.3125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3551637530326843,
+      "epoch": 3.1707317073170733,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0931132510304451,
+      "kl": 6.375309385475703e-05,
+      "learning_rate": 8.66112032384275e-07,
+      "loss": -0.04869828745722771,
+      "num_tokens": 3211594.0,
+      "reward": 0.35241150856018066,
+      "reward_std": 0.2379828542470932,
+      "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+      "rewards/true_env_reward_fn/std": 0.2379828542470932,
       "step": 130,
-      "step_time": 4.251137947001553
+      "step_time": 15.15810051499966
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 67.0,
-      "completions/mean_terminated_length": 67.0,
-      "completions/min_length": 44.0,
-      "completions/min_terminated_length": 44.0,
-      "entropy": 1.0767641067504883,
-      "epoch": 1.065040650406504,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15786457061767578,
-      "kl": 1.8847958926926367e-05,
-      "learning_rate": 6.548529203491875e-07,
-      "loss": -0.0026272237300872803,
-      "num_tokens": 539269.0,
-      "reward": 0.536803662776947,
-      "reward_std": 0.30375123023986816,
-      "rewards/true_env_reward_fn/mean": 0.536803662776947,
-      "rewards/true_env_reward_fn/std": 0.30375123023986816,
+      "completions/max_length": 199.0,
+      "completions/max_terminated_length": 199.0,
+      "completions/mean_length": 66.6875,
+      "completions/mean_terminated_length": 66.6875,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 1.212640792131424,
+      "epoch": 3.1951219512195124,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.0767395943403244,
+      "kl": 4.670183352573076e-05,
+      "learning_rate": 8.631995777392644e-07,
+      "loss": -0.02368815243244171,
+      "num_tokens": 3242883.0,
+      "reward": 0.3383604884147644,
+      "reward_std": 0.31325310468673706,
+      "rewards/true_env_reward_fn/mean": 0.338360458612442,
+      "rewards/true_env_reward_fn/std": 0.31325310468673706,
       "step": 131,
-      "step_time": 3.7980547870010923
+      "step_time": 20.109428818001106
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/max_terminated_length": 85.0,
-      "completions/mean_length": 64.25,
-      "completions/mean_terminated_length": 64.25,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "entropy": 1.3295028805732727,
-      "epoch": 1.0731707317073171,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.1881481558084488,
-      "kl": 2.0969039724150207e-05,
-      "learning_rate": 6.519242088113085e-07,
-      "loss": 0.08431969583034515,
-      "num_tokens": 545691.0,
-      "reward": 0.24590599536895752,
-      "reward_std": 0.2047487199306488,
-      "rewards/true_env_reward_fn/mean": 0.24590599536895752,
-      "rewards/true_env_reward_fn/std": 0.2047487199306488,
+      "completions/max_length": 107.0,
+      "completions/max_terminated_length": 107.0,
+      "completions/mean_length": 59.520835876464844,
+      "completions/mean_terminated_length": 59.520835876464844,
+      "completions/min_length": 23.0,
+      "completions/min_terminated_length": 23.0,
+      "entropy": 1.3366018533706665,
+      "epoch": 3.2195121951219514,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.11203660070896149,
+      "kl": 6.134294108051108e-05,
+      "learning_rate": 8.602607968003934e-07,
+      "loss": -0.03865987807512283,
+      "num_tokens": 3268372.0,
+      "reward": 0.440601110458374,
+      "reward_std": 0.336189866065979,
+      "rewards/true_env_reward_fn/mean": 0.440601110458374,
+      "rewards/true_env_reward_fn/std": 0.336189866065979,
       "step": 132,
-      "step_time": 4.361092664001262
+      "step_time": 10.12403799699996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 72.0,
-      "completions/max_terminated_length": 72.0,
-      "completions/mean_length": 57.5,
-      "completions/mean_terminated_length": 57.5,
-      "completions/min_length": 41.0,
-      "completions/min_terminated_length": 41.0,
-      "entropy": 1.2131375670433044,
-      "epoch": 1.08130081300813,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.13930389285087585,
-      "kl": 1.1046585314034019e-05,
-      "learning_rate": 6.489729398111058e-07,
-      "loss": -0.03801802545785904,
-      "num_tokens": 550295.0,
-      "reward": 0.3215479254722595,
-      "reward_std": 0.1736886352300644,
-      "rewards/true_env_reward_fn/mean": 0.3215479254722595,
-      "rewards/true_env_reward_fn/std": 0.17368865013122559,
+      "completions/max_length": 138.0,
+      "completions/max_terminated_length": 138.0,
+      "completions/mean_length": 61.41666793823242,
+      "completions/mean_terminated_length": 61.41666793823242,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "entropy": 1.2847907543182373,
+      "epoch": 3.2439024390243905,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10071831941604614,
+      "kl": 6.808681609982159e-05,
+      "learning_rate": 8.572959025833573e-07,
+      "loss": 0.0024422593414783478,
+      "num_tokens": 3291888.0,
+      "reward": 0.3618060350418091,
+      "reward_std": 0.26743030548095703,
+      "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+      "rewards/true_env_reward_fn/std": 0.26743027567863464,
       "step": 133,
-      "step_time": 3.372364626999115
+      "step_time": 10.396350653999434
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/max_terminated_length": 70.0,
-      "completions/mean_length": 63.375,
-      "completions/mean_terminated_length": 63.375,
-      "completions/min_length": 54.0,
-      "completions/min_terminated_length": 54.0,
-      "entropy": 1.2786019444465637,
-      "epoch": 1.089430894308943,
+      "completions/max_length": 116.0,
+      "completions/max_terminated_length": 116.0,
+      "completions/mean_length": 67.4375,
+      "completions/mean_terminated_length": 67.4375,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.1504567563533783,
+      "epoch": 3.2682926829268295,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12932609021663666,
-      "kl": 1.340499647994875e-05,
-      "learning_rate": 6.459993776071815e-07,
-      "loss": 0.029022663831710815,
-      "num_tokens": 553826.0,
-      "reward": 0.4830188751220703,
-      "reward_std": 0.29014864563941956,
-      "rewards/true_env_reward_fn/mean": 0.4830188751220703,
-      "rewards/true_env_reward_fn/std": 0.29014864563941956,
+      "grad_norm": 0.060020897537469864,
+      "kl": 3.462390031927498e-05,
+      "learning_rate": 8.543051099966557e-07,
+      "loss": 0.04882139340043068,
+      "num_tokens": 3317125.0,
+      "reward": 0.5031180381774902,
+      "reward_std": 0.2628377676010132,
+      "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+      "rewards/true_env_reward_fn/std": 0.2628377676010132,
       "step": 134,
-      "step_time": 3.215292060998763
+      "step_time": 10.961974539000039
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 75.0,
-      "completions/max_terminated_length": 75.0,
-      "completions/mean_length": 49.375,
-      "completions/mean_terminated_length": 49.375,
-      "completions/min_length": 28.0,
-      "completions/min_terminated_length": 28.0,
-      "entropy": 0.9003906548023224,
-      "epoch": 1.0975609756097562,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.15958240628242493,
-      "kl": 2.838099044311093e-05,
-      "learning_rate": 6.430037884542861e-07,
-      "loss": 0.11459673941135406,
-      "num_tokens": 557217.0,
-      "reward": 0.494448184967041,
-      "reward_std": 0.3076546788215637,
-      "rewards/true_env_reward_fn/mean": 0.494448184967041,
-      "rewards/true_env_reward_fn/std": 0.3076546788215637,
+      "completions/max_length": 128.0,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 65.45833587646484,
+      "completions/mean_terminated_length": 65.45833587646484,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "entropy": 1.3224314153194427,
+      "epoch": 3.292682926829268,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.10321197658777237,
+      "kl": 6.988596032897476e-05,
+      "learning_rate": 8.51288635826016e-07,
+      "loss": 0.011862488463521004,
+      "num_tokens": 3347059.0,
+      "reward": 0.39905214309692383,
+      "reward_std": 0.31803515553474426,
+      "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+      "rewards/true_env_reward_fn/std": 0.31803515553474426,
       "step": 135,
-      "step_time": 3.500462582000182
+      "step_time": 11.779171687000144
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
       "completions/clipped_ratio": 0.0,
       "completions/max_length": 93.0,
       "completions/max_terminated_length": 93.0,
-      "completions/mean_length": 57.5,
-      "completions/mean_terminated_length": 57.5,
-      "completions/min_length": 38.0,
-      "completions/min_terminated_length": 38.0,
-      "entropy": 1.2928712964057922,
-      "epoch": 1.1056910569105691,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.23077522218227386,
-      "kl": 2.251418845844455e-05,
-      "learning_rate": 6.399864405794782e-07,
-      "loss": -0.05874824523925781,
-      "num_tokens": 562421.0,
-      "reward": 0.2385583370923996,
-      "reward_std": 0.23380905389785767,
-      "rewards/true_env_reward_fn/mean": 0.2385583370923996,
-      "rewards/true_env_reward_fn/std": 0.23380906879901886,
+      "completions/mean_length": 60.458335876464844,
+      "completions/mean_terminated_length": 60.458335876464844,
+      "completions/min_length": 29.0,
+      "completions/min_terminated_length": 29.0,
+      "entropy": 1.1519232094287872,
+      "epoch": 3.317073170731707,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.085839182138443,
+      "kl": 5.779342154710321e-05,
+      "learning_rate": 8.482466987186785e-07,
+      "loss": 0.05022352561354637,
+      "num_tokens": 3370225.0,
+      "reward": 0.4742569923400879,
+      "reward_std": 0.3171122074127197,
+      "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+      "rewards/true_env_reward_fn/std": 0.3171122074127197,
       "step": 136,
-      "step_time": 4.208805245998519
+      "step_time": 8.779588141000204
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 89.0,
-      "completions/max_terminated_length": 89.0,
-      "completions/mean_length": 63.25,
-      "completions/mean_terminated_length": 63.25,
-      "completions/min_length": 45.0,
-      "completions/min_terminated_length": 45.0,
-      "entropy": 0.8736326098442078,
-      "epoch": 1.113821138211382,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 8.871674071997404e-05,
-      "kl": 1.1485328741400735e-05,
-      "learning_rate": 6.369476041581066e-07,
-      "loss": 5.747077125306532e-07,
-      "num_tokens": 566387.0,
-      "reward": 0.4902166724205017,
-      "reward_std": 0.038254011422395706,
-      "rewards/true_env_reward_fn/mean": 0.4902166724205017,
-      "rewards/true_env_reward_fn/std": 0.038254011422395706,
+      "completions/max_length": 130.0,
+      "completions/max_terminated_length": 130.0,
+      "completions/mean_length": 65.6875,
+      "completions/mean_terminated_length": 65.6875,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "entropy": 1.377644658088684,
+      "epoch": 3.341463414634146,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.05921673774719238,
+      "kl": 3.667381497507449e-05,
+      "learning_rate": 8.451795191675487e-07,
+      "loss": 0.020366013050079346,
+      "num_tokens": 3399578.0,
+      "reward": 0.3279460370540619,
+      "reward_std": 0.4147447645664215,
+      "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+      "rewards/true_env_reward_fn/std": 0.4147447645664215,
       "step": 137,
-      "step_time": 3.981489739000608
+      "step_time": 11.74765996799988
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 60.25,
-      "completions/mean_terminated_length": 60.25,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.1538971662521362,
-      "epoch": 1.1219512195121952,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14676779508590698,
-      "kl": 1.1651037766569061e-05,
-      "learning_rate": 6.338875512896188e-07,
-      "loss": 0.1347643882036209,
-      "num_tokens": 569341.0,
-      "reward": 0.43844783306121826,
-      "reward_std": 0.16067014634609222,
-      "rewards/true_env_reward_fn/mean": 0.43844783306121826,
-      "rewards/true_env_reward_fn/std": 0.16067016124725342,
+      "completions/max_length": 112.0,
+      "completions/max_terminated_length": 112.0,
+      "completions/mean_length": 61.583335876464844,
+      "completions/mean_terminated_length": 61.583335876464844,
+      "completions/min_length": 26.0,
+      "completions/min_terminated_length": 26.0,
+      "entropy": 1.2057753205299377,
+      "epoch": 3.3658536585365852,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08148445934057236,
+      "kl": 3.0601177968492266e-05,
+      "learning_rate": 8.420873194952152e-07,
+      "loss": 0.005453992635011673,
+      "num_tokens": 3417734.0,
+      "reward": 0.5946073532104492,
+      "reward_std": 0.25090643763542175,
+      "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+      "rewards/true_env_reward_fn/std": 0.25090643763542175,
       "step": 138,
-      "step_time": 3.689221037999232
+      "step_time": 8.8135579650002
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 76.0,
-      "completions/max_terminated_length": 76.0,
-      "completions/mean_length": 54.625,
-      "completions/mean_terminated_length": 54.625,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "entropy": 1.1961707472801208,
-      "epoch": 1.1300813008130082,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.16650564968585968,
-      "kl": 1.4349476259667426e-05,
-      "learning_rate": 6.308065559731976e-07,
-      "loss": 0.007910434156656265,
-      "num_tokens": 574046.0,
-      "reward": 0.4596000015735626,
-      "reward_std": 0.07715634256601334,
-      "rewards/true_env_reward_fn/mean": 0.4596000015735626,
-      "rewards/true_env_reward_fn/std": 0.07715633511543274,
+      "completions/max_length": 105.0,
+      "completions/max_terminated_length": 105.0,
+      "completions/mean_length": 66.47917175292969,
+      "completions/mean_terminated_length": 66.47917175292969,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 1.3243012130260468,
+      "epoch": 3.3902439024390243,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06626639515161514,
+      "kl": 3.7586045436910354e-05,
+      "learning_rate": 8.389703238378338e-07,
+      "loss": -0.003325441852211952,
+      "num_tokens": 3441749.0,
+      "reward": 0.48056626319885254,
+      "reward_std": 0.2497076690196991,
+      "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+      "rewards/true_env_reward_fn/std": 0.2497076541185379,
       "step": 139,
-      "step_time": 3.6711935700004688
+      "step_time": 9.997661417000472
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 57.875,
-      "completions/mean_terminated_length": 57.875,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.2013322114944458,
-      "epoch": 1.1382113821138211,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 189.0,
+      "completions/mean_length": 75.79167175292969,
+      "completions/mean_terminated_length": 75.79167175292969,
+      "completions/min_length": 40.0,
+      "completions/min_terminated_length": 40.0,
+      "entropy": 1.2918364107608795,
+      "epoch": 3.4146341463414633,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14415834844112396,
-      "kl": 1.4664098671346437e-05,
-      "learning_rate": 6.277048940832264e-07,
-      "loss": -0.016162052750587463,
-      "num_tokens": 576769.0,
-      "reward": 0.6152583360671997,
-      "reward_std": 0.07727260142564774,
-      "rewards/true_env_reward_fn/mean": 0.6152583360671997,
-      "rewards/true_env_reward_fn/std": 0.07727260142564774,
+      "grad_norm": 0.0600166842341423,
+      "kl": 3.408677366678603e-05,
+      "learning_rate": 8.358287581288822e-07,
+      "loss": -0.002709554508328438,
+      "num_tokens": 3473139.0,
+      "reward": 0.38171443343162537,
+      "reward_std": 0.2058144509792328,
+      "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+      "rewards/true_env_reward_fn/std": 0.2058144509792328,
       "step": 140,
-      "step_time": 3.5191362610003125
+      "step_time": 14.679971276999822
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 52.0,
-      "completions/max_terminated_length": 52.0,
-      "completions/mean_length": 44.75,
-      "completions/mean_terminated_length": 44.75,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.0287770330905914,
-      "epoch": 1.146341463414634,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 9.892051457427442e-05,
-      "kl": 1.1797974821092794e-05,
-      "learning_rate": 6.245828433445872e-07,
-      "loss": 5.92092192164273e-07,
-      "num_tokens": 578843.0,
-      "reward": 0.6387845277786255,
-      "reward_std": 0.13867565989494324,
-      "rewards/true_env_reward_fn/mean": 0.6387845277786255,
-      "rewards/true_env_reward_fn/std": 0.13867565989494324,
+      "completions/max_length": 97.0,
+      "completions/max_terminated_length": 97.0,
+      "completions/mean_length": 62.6875,
+      "completions/mean_terminated_length": 62.6875,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "entropy": 1.32420814037323,
+      "epoch": 3.4390243902439024,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.08056586235761642,
+      "kl": 4.234552670823177e-05,
+      "learning_rate": 8.326628500827825e-07,
+      "loss": 0.019535928964614868,
+      "num_tokens": 3499324.0,
+      "reward": 0.4587298631668091,
+      "reward_std": 0.3119663596153259,
+      "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+      "rewards/true_env_reward_fn/std": 0.3119663596153259,
       "step": 141,
-      "step_time": 2.5015027329991426
+      "step_time": 11.64747691499997
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.0,
-      "completions/max_terminated_length": 65.0,
-      "completions/mean_length": 50.75,
-      "completions/mean_terminated_length": 50.75,
+      "completions/max_length": 122.0,
+      "completions/max_terminated_length": 122.0,
+      "completions/mean_length": 69.27083587646484,
+      "completions/mean_terminated_length": 69.27083587646484,
       "completions/min_length": 37.0,
       "completions/min_terminated_length": 37.0,
-      "entropy": 1.1590029001235962,
-      "epoch": 1.1544715447154472,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.160966694355011,
-      "kl": 1.4735675904375967e-05,
-      "learning_rate": 6.214406833077937e-07,
-      "loss": 0.0170527845621109,
-      "num_tokens": 583201.0,
-      "reward": 0.36017733812332153,
-      "reward_std": 0.3556094467639923,
-      "rewards/true_env_reward_fn/mean": 0.36017733812332153,
-      "rewards/true_env_reward_fn/std": 0.3556094467639923,
+      "entropy": 1.3127666413784027,
+      "epoch": 3.4634146341463414,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07890015095472336,
+      "kl": 4.281656902094255e-05,
+      "learning_rate": 8.294728291783965e-07,
+      "loss": -0.034988921135663986,
+      "num_tokens": 3516425.0,
+      "reward": 0.6331583261489868,
+      "reward_std": 0.2317410111427307,
+      "rewards/true_env_reward_fn/mean": 0.633158266544342,
+      "rewards/true_env_reward_fn/std": 0.23174098134040833,
       "step": 142,
-      "step_time": 3.2783409929998015
+      "step_time": 8.380270293999729
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/max_terminated_length": 83.0,
-      "completions/mean_length": 67.0,
-      "completions/mean_terminated_length": 67.0,
-      "completions/min_length": 59.0,
-      "completions/min_terminated_length": 59.0,
-      "entropy": 1.1985241174697876,
-      "epoch": 1.1626016260162602,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.12272457778453827,
-      "kl": 1.7849098185251933e-05,
-      "learning_rate": 6.182786953239593e-07,
-      "loss": -0.0016125142574310303,
-      "num_tokens": 587317.0,
-      "reward": 0.34745320677757263,
-      "reward_std": 0.3954337239265442,
-      "rewards/true_env_reward_fn/mean": 0.34745320677757263,
-      "rewards/true_env_reward_fn/std": 0.3954337537288666,
+      "completions/max_length": 142.0,
+      "completions/max_terminated_length": 142.0,
+      "completions/mean_length": 73.97917175292969,
+      "completions/mean_terminated_length": 73.97917175292969,
+      "completions/min_length": 44.0,
+      "completions/min_terminated_length": 44.0,
+      "entropy": 1.323029786348343,
+      "epoch": 3.4878048780487805,
+      "frac_reward_zero_std": 0.1666666716337204,
+      "grad_norm": 0.08455090969800949,
+      "kl": 6.877856139908545e-05,
+      "learning_rate": 8.262589266423908e-07,
+      "loss": 0.06993371993303299,
+      "num_tokens": 3542912.0,
+      "reward": 0.41727983951568604,
+      "reward_std": 0.23754946887493134,
+      "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+      "rewards/true_env_reward_fn/std": 0.23754946887493134,
       "step": 143,
-      "step_time": 3.9932043310000154
+      "step_time": 11.716556537000088
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 48.0,
-      "completions/max_terminated_length": 48.0,
-      "completions/mean_length": 43.75,
-      "completions/mean_terminated_length": 43.75,
-      "completions/min_length": 35.0,
-      "completions/min_terminated_length": 35.0,
-      "entropy": 1.1116944551467896,
-      "epoch": 1.170731707317073,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.18435250222682953,
-      "kl": 1.014559029499651e-05,
-      "learning_rate": 6.150971625196048e-07,
-      "loss": 0.009793907403945923,
-      "num_tokens": 590191.0,
-      "reward": 0.4938516616821289,
-      "reward_std": 0.03703190013766289,
-      "rewards/true_env_reward_fn/mean": 0.4938516616821289,
-      "rewards/true_env_reward_fn/std": 0.037031903862953186,
+      "completions/max_length": 104.0,
+      "completions/max_terminated_length": 104.0,
+      "completions/mean_length": 63.3125,
+      "completions/mean_terminated_length": 63.3125,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.3465435802936554,
+      "epoch": 3.5121951219512195,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.09822116792201996,
+      "kl": 8.00468278612243e-05,
+      "learning_rate": 8.230213754324772e-07,
+      "loss": 0.07691670209169388,
+      "num_tokens": 3569575.0,
+      "reward": 0.28445714712142944,
+      "reward_std": 0.33810389041900635,
+      "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+      "rewards/true_env_reward_fn/std": 0.33810392022132874,
       "step": 144,
-      "step_time": 2.3663663690022076
+      "step_time": 10.67718802499985
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.0,
-      "completions/max_terminated_length": 87.0,
-      "completions/mean_length": 53.0,
-      "completions/mean_terminated_length": 53.0,
-      "completions/min_length": 37.0,
-      "completions/min_terminated_length": 37.0,
-      "entropy": 1.413200855255127,
-      "epoch": 1.1788617886178863,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.14336225390434265,
-      "kl": 2.0541991034406237e-05,
-      "learning_rate": 6.118963697713078e-07,
-      "loss": -0.013927727937698364,
-      "num_tokens": 593671.0,
-      "reward": 0.4619143605232239,
-      "reward_std": 0.3773181140422821,
-      "rewards/true_env_reward_fn/mean": 0.4619143605232239,
-      "rewards/true_env_reward_fn/std": 0.3773181140422821,
+      "completions/max_length": 126.0,
+      "completions/max_terminated_length": 126.0,
+      "completions/mean_length": 74.375,
+      "completions/mean_terminated_length": 74.375,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.3208706378936768,
+      "epoch": 3.5365853658536586,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06776741147041321,
+      "kl": 4.5862597744417144e-05,
+      "learning_rate": 8.19760410220527e-07,
+      "loss": -0.014808554202318192,
+      "num_tokens": 3589641.0,
+      "reward": 0.5829761028289795,
+      "reward_std": 0.21224236488342285,
+      "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+      "rewards/true_env_reward_fn/std": 0.21224237978458405,
       "step": 145,
-      "step_time": 3.9730388410007436
+      "step_time": 8.610043666999445
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 128.0,
-      "completions/max_terminated_length": 128.0,
-      "completions/mean_length": 68.625,
-      "completions/mean_terminated_length": 68.625,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "entropy": 1.08676016330719,
-      "epoch": 1.1869918699186992,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.15555191040039062,
-      "kl": 1.6947700260061538e-05,
-      "learning_rate": 6.086766036801937e-07,
-      "loss": -0.139797180891037,
-      "num_tokens": 601612.0,
-      "reward": 0.3831036686897278,
-      "reward_std": 0.09242849797010422,
-      "rewards/true_env_reward_fn/mean": 0.3831036686897278,
-      "rewards/true_env_reward_fn/std": 0.09242849797010422,
+      "completions/max_length": 180.0,
+      "completions/max_terminated_length": 180.0,
+      "completions/mean_length": 72.1875,
+      "completions/mean_terminated_length": 72.1875,
+      "completions/min_length": 31.0,
+      "completions/min_terminated_length": 31.0,
+      "entropy": 1.2934723794460297,
+      "epoch": 3.5609756097560976,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.05893269553780556,
+      "kl": 3.648432630143361e-05,
+      "learning_rate": 8.164762673755609e-07,
+      "loss": 0.023374930024147034,
+      "num_tokens": 3615570.0,
+      "reward": 0.47375163435935974,
+      "reward_std": 0.16054874658584595,
+      "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+      "rewards/true_env_reward_fn/std": 0.16054873168468475,
       "step": 146,
-      "step_time": 6.323679949000507
+      "step_time": 13.649344002000362
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 116.0,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 67.5,
-      "completions/mean_terminated_length": 67.5,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 1.5055813789367676,
-      "epoch": 1.1951219512195121,
+      "completions/max_length": 125.0,
+      "completions/max_terminated_length": 125.0,
+      "completions/mean_length": 72.1875,
+      "completions/mean_terminated_length": 72.1875,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "entropy": 1.3180726766586304,
+      "epoch": 3.5853658536585367,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2397669553756714,
-      "kl": 2.704876442294335e-05,
-      "learning_rate": 6.054381525462745e-07,
-      "loss": 0.2738838493824005,
-      "num_tokens": 606712.0,
-      "reward": 0.25339600443840027,
-      "reward_std": 0.3023079037666321,
-      "rewards/true_env_reward_fn/mean": 0.25339600443840027,
-      "rewards/true_env_reward_fn/std": 0.3023079037666321,
+      "grad_norm": 0.08518138527870178,
+      "kl": 6.788871905882843e-05,
+      "learning_rate": 8.131691849466152e-07,
+      "loss": -0.04987313598394394,
+      "num_tokens": 3637475.0,
+      "reward": 0.5195532441139221,
+      "reward_std": 0.26043611764907837,
+      "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+      "rewards/true_env_reward_fn/std": 0.26043611764907837,
       "step": 147,
-      "step_time": 5.185072233998653
+      "step_time": 11.702765863000877
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 47.25,
-      "completions/mean_terminated_length": 47.25,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 1.135968267917633,
-      "epoch": 1.203252032520325,
+      "completions/max_length": 168.0,
+      "completions/max_terminated_length": 168.0,
+      "completions/mean_length": 73.83333587646484,
+      "completions/mean_terminated_length": 73.83333587646484,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
+      "entropy": 1.167496383190155,
+      "epoch": 3.6097560975609757,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2614514231681824,
-      "kl": 3.613240005506668e-05,
-      "learning_rate": 6.021813063426323e-07,
-      "loss": 0.10286401212215424,
-      "num_tokens": 610566.0,
-      "reward": 0.31031692028045654,
-      "reward_std": 0.3124054968357086,
-      "rewards/true_env_reward_fn/mean": 0.31031692028045654,
-      "rewards/true_env_reward_fn/std": 0.312405526638031,
+      "grad_norm": 0.07978484779596329,
+      "kl": 6.430712710425723e-05,
+      "learning_rate": 8.098394026454884e-07,
+      "loss": 0.024383332580327988,
+      "num_tokens": 3663171.0,
+      "reward": 0.4524516761302948,
+      "reward_std": 0.2587544322013855,
+      "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+      "rewards/true_env_reward_fn/std": 0.2587544322013855,
       "step": 148,
-      "step_time": 3.2177847610000754
+      "step_time": 13.306644664000487
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 96.0,
-      "completions/max_terminated_length": 96.0,
-      "completions/mean_length": 52.75,
-      "completions/mean_terminated_length": 52.75,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 1.4589928984642029,
-      "epoch": 1.2113821138211383,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.2019941508769989,
-      "kl": 2.1841721718374174e-05,
-      "learning_rate": 5.989063566894572e-07,
-      "loss": 0.010915875434875488,
-      "num_tokens": 615716.0,
-      "reward": 0.31711751222610474,
-      "reward_std": 0.13289952278137207,
-      "rewards/true_env_reward_fn/mean": 0.31711751222610474,
-      "rewards/true_env_reward_fn/std": 0.13289952278137207,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 65.875,
+      "completions/mean_terminated_length": 65.875,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "entropy": 1.155810385942459,
+      "epoch": 3.6341463414634148,
+      "frac_reward_zero_std": 0.6666666865348816,
+      "grad_norm": 0.046879056841135025,
+      "kl": 4.4023097871104255e-05,
+      "learning_rate": 8.064871618293645e-07,
+      "loss": -0.01477135717868805,
+      "num_tokens": 3683813.0,
+      "reward": 0.6097190380096436,
+      "reward_std": 0.17910261452198029,
+      "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+      "rewards/true_env_reward_fn/std": 0.17910261452198029,
       "step": 149,
-      "step_time": 4.3804878079990885
+      "step_time": 9.446422488999815
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -4188,32 +4188,32 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.0,
-      "completions/max_terminated_length": 79.0,
-      "completions/mean_length": 67.375,
-      "completions/mean_terminated_length": 67.375,
-      "completions/min_length": 52.0,
-      "completions/min_terminated_length": 52.0,
-      "entropy": 1.1892729997634888,
-      "epoch": 1.2195121951219512,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 7.835455471649766e-05,
-      "kl": 1.3420096820482286e-05,
-      "learning_rate": 5.956135968279332e-07,
-      "loss": 6.646802717114042e-07,
-      "num_tokens": 619439.0,
-      "reward": 0.6557307243347168,
-      "reward_std": 0.2151959389448166,
-      "rewards/true_env_reward_fn/mean": 0.6557307243347168,
-      "rewards/true_env_reward_fn/std": 0.21519595384597778,
+      "completions/max_length": 162.0,
+      "completions/max_terminated_length": 162.0,
+      "completions/mean_length": 72.8125,
+      "completions/mean_terminated_length": 72.8125,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 1.3542745113372803,
+      "epoch": 3.658536585365854,
+      "frac_reward_zero_std": 0.3333333432674408,
+      "grad_norm": 0.06843585520982742,
+      "kl": 3.291011944384081e-05,
+      "learning_rate": 8.03112705483319e-07,
+      "loss": 0.009258950129151344,
+      "num_tokens": 3702516.0,
+      "reward": 0.5345131754875183,
+      "reward_std": 0.22612926363945007,
+      "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+      "rewards/true_env_reward_fn/std": 0.22612926363945007,
       "step": 150,
-      "step_time": 3.63938895299907
+      "step_time": 10.538116119999813
     }
   ],
   "logging_steps": 1,
-  "max_steps": 369,
-  "num_input_tokens_seen": 619439,
-  "num_train_epochs": 3,
+  "max_steps": 410,
+  "num_input_tokens_seen": 3702516,
+  "num_train_epochs": 10,
   "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -4228,7 +4228,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 12,
   "trial_name": null,
   "trial_params": null
 }