checkpoint step 150

checkpoint-150/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a1a5c3c3da55d9a821abb24b055007ec7d5877b0bed4c0637df2b3960eeea78
+oid sha256:61441c27c8e2946fac3d6d724768c81e03d3c7152dc542c9aebcc45884960470
 size 80792880

checkpoint-150/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c9a96f491cbb9427662347239b8c973c284ee4eb188879c6e7cfee8b553755d
+oid sha256:6b761887b9802d820ad2887652d84840fcfc58c451300200d7f80249af86ffa2
 size 161816251

checkpoint-150/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da102c6ca406d979362d2525e516304da9e042a41e98efeb30694dff44442397
+oid sha256:e227f117fd587201c8f558fa675e184363377e29997075648dd2577c8c0a5708
 size 14645

checkpoint-150/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea76193194eb47ac3392cf289f34baa99eccbe563ba9402f128f69b35ba11721
+oid sha256:4f5dd9010deb31119d8d374692c18051d183d660658e91856f8ce0f6358460d6
 size 1465

checkpoint-150/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.00646551724137931,
+  "epoch": 75.0,
   "eval_steps": 500,
   "global_step": 150,
   "is_hyper_param_search": false,
@@ -15,26 +15,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 130.0,
-      "completions/max_terminated_length": 130.0,
-      "completions/mean_length": 68.5,
-      "completions/mean_terminated_length": 68.5,
-      "completions/min_length": 26.8,
-      "completions/min_terminated_length": 26.8,
-      "entropy": 0.25826080311089755,
-      "epoch": 0.00021551724137931034,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.28125,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 193.0,
+      "completions/max_terminated_length": 163.0,
+      "completions/mean_length": 103.975,
+      "completions/mean_terminated_length": 90.39881134033203,
+      "completions/min_length": 17.6,
+      "completions/min_terminated_length": 17.6,
+      "entropy": 0.1820149033330381,
+      "epoch": 2.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.25,
       "learning_rate": 1.0000000000000002e-06,
-      "loss": 0.10238287448883057,
-      "num_tokens": 12064.0,
-      "reward": 0.3290319949388504,
-      "reward_std": 0.40028320252895355,
-      "rewards/reward_fn/mean": 0.3290319949388504,
-      "rewards/reward_fn/std": 0.400283208489418,
+      "loss": 0.0794088900089264,
+      "num_tokens": 14355.0,
+      "reward": 0.15020800828933717,
+      "reward_std": 0.6376187483081595,
+      "rewards/reward_fn/mean": 0.15020800828933717,
+      "rewards/reward_fn/std": 0.6376187764341011,
       "step": 5,
-      "step_time": 22.194597821800016
+      "step_time": 30.522303848797673
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -42,26 +42,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.6,
-      "completions/max_terminated_length": 87.6,
-      "completions/mean_length": 48.525,
-      "completions/mean_terminated_length": 48.525,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
-      "entropy": 0.2673827801831067,
-      "epoch": 0.0004310344827586207,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.484375,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 151.2,
+      "completions/max_terminated_length": 141.6,
+      "completions/mean_length": 80.075,
+      "completions/mean_terminated_length": 77.78928833007812,
+      "completions/min_length": 17.8,
+      "completions/min_terminated_length": 17.8,
+      "entropy": 0.16221200795844198,
+      "epoch": 5.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.228515625,
       "learning_rate": 2.25e-06,
-      "loss": 0.05849265456199646,
-      "num_tokens": 23089.0,
-      "reward": 0.45969198942184447,
-      "reward_std": 0.2795014828443527,
-      "rewards/reward_fn/mean": 0.45969198942184447,
-      "rewards/reward_fn/std": 0.27950150668621065,
+      "loss": 0.07169516086578369,
+      "num_tokens": 27462.0,
+      "reward": 0.4012819856405258,
+      "reward_std": 0.4128362699819263,
+      "rewards/reward_fn/mean": 0.4012819856405258,
+      "rewards/reward_fn/std": 0.4128363010211615,
       "step": 10,
-      "step_time": 16.331819552399928
+      "step_time": 25.01976956339822
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -69,26 +69,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.6,
-      "completions/max_terminated_length": 85.6,
-      "completions/mean_length": 47.35,
-      "completions/mean_terminated_length": 47.35,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 0.20174795808270574,
-      "epoch": 0.000646551724137931,
-      "frac_reward_zero_std": 0.4,
-      "grad_norm": 0.66015625,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 185.6,
+      "completions/max_terminated_length": 171.6,
+      "completions/mean_length": 83.65,
+      "completions/mean_terminated_length": 79.14285736083984,
+      "completions/min_length": 16.6,
+      "completions/min_terminated_length": 16.6,
+      "entropy": 0.14225535104051232,
+      "epoch": 7.5,
+      "frac_reward_zero_std": 0.3,
+      "grad_norm": 0.265625,
       "learning_rate": 3.5e-06,
-      "loss": -0.03555725216865539,
-      "num_tokens": 34239.0,
-      "reward": 0.5545999944210053,
-      "reward_std": 0.32832055240869523,
-      "rewards/reward_fn/mean": 0.5545999944210053,
-      "rewards/reward_fn/std": 0.3283205583691597,
+      "loss": 0.0717179834842682,
+      "num_tokens": 40740.0,
+      "reward": 0.10207997858524323,
+      "reward_std": 0.7454913818277419,
+      "rewards/reward_fn/mean": 0.10207997858524323,
+      "rewards/reward_fn/std": 0.7454914333298802,
       "step": 15,
-      "step_time": 16.44816417159991
+      "step_time": 29.626422570000432
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -96,26 +96,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 98.2,
-      "completions/max_terminated_length": 98.2,
-      "completions/mean_length": 53.6,
-      "completions/mean_terminated_length": 53.6,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 0.2620095924474299,
-      "epoch": 0.0008620689655172414,
-      "frac_reward_zero_std": 0.2,
-      "grad_norm": 0.5390625,
+      "completions/clipped_ratio": 0.1,
+      "completions/max_length": 202.0,
+      "completions/max_terminated_length": 163.0,
+      "completions/mean_length": 98.85,
+      "completions/mean_terminated_length": 81.79285888671875,
+      "completions/min_length": 19.2,
+      "completions/min_terminated_length": 19.2,
+      "entropy": 0.1534867493668571,
+      "epoch": 10.0,
+      "frac_reward_zero_std": 0.4,
+      "grad_norm": 0.2431640625,
       "learning_rate": 4.75e-06,
-      "loss": 0.017401468753814698,
-      "num_tokens": 45399.0,
-      "reward": 0.4466240078210831,
-      "reward_std": 0.27573536019772293,
-      "rewards/reward_fn/mean": 0.4466240078210831,
-      "rewards/reward_fn/std": 0.27573537137359383,
+      "loss": 0.09726614952087402,
+      "num_tokens": 54862.0,
+      "reward": 0.25181599259376525,
+      "reward_std": 0.6045640033902601,
+      "rewards/reward_fn/mean": 0.25181599259376525,
+      "rewards/reward_fn/std": 0.6045640454394743,
       "step": 20,
-      "step_time": 17.988385831799953
+      "step_time": 31.625062462999267
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -123,26 +123,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.8,
-      "completions/max_terminated_length": 123.8,
-      "completions/mean_length": 59.15,
-      "completions/mean_terminated_length": 59.15,
-      "completions/min_length": 21.8,
-      "completions/min_terminated_length": 21.8,
-      "entropy": 0.33701689867302775,
-      "epoch": 0.0010775862068965517,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.74609375,
-      "learning_rate": 4.965517241379311e-06,
-      "loss": -0.02674364447593689,
-      "num_tokens": 56993.0,
-      "reward": 0.5006439983844757,
-      "reward_std": 0.2351181447505951,
-      "rewards/reward_fn/mean": 0.5006439983844757,
-      "rewards/reward_fn/std": 0.23511814773082734,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 233.6,
+      "completions/max_terminated_length": 194.4,
+      "completions/mean_length": 99.525,
+      "completions/mean_terminated_length": 77.78690643310547,
+      "completions/min_length": 18.8,
+      "completions/min_terminated_length": 18.8,
+      "entropy": 0.1538564210291952,
+      "epoch": 12.5,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.0,
+      "learning_rate": 4.981481481481482e-06,
+      "loss": 0.13929661512374877,
+      "num_tokens": 69039.0,
+      "reward": 0.0006859898567199707,
+      "reward_std": 1.0108385920524596,
+      "rewards/reward_fn/mean": 0.0006859898567199707,
+      "rewards/reward_fn/std": 1.0108386158943177,
       "step": 25,
-      "step_time": 21.11461545459997
+      "step_time": 35.65783783460065
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -150,26 +150,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 110.2,
-      "completions/max_terminated_length": 110.2,
-      "completions/mean_length": 47.95,
-      "completions/mean_terminated_length": 47.95,
-      "completions/min_length": 15.8,
-      "completions/min_terminated_length": 15.8,
-      "entropy": 0.24504410615190864,
-      "epoch": 0.001293103448275862,
-      "frac_reward_zero_std": 0.4,
-      "grad_norm": 0.92578125,
-      "learning_rate": 4.922413793103449e-06,
-      "loss": -0.05023183822631836,
-      "num_tokens": 67659.0,
-      "reward": 0.6457239985466003,
-      "reward_std": 0.11179900387069211,
-      "rewards/reward_fn/mean": 0.6457239985466003,
-      "rewards/reward_fn/std": 0.11179901438299567,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 181.8,
+      "completions/max_terminated_length": 153.0,
+      "completions/mean_length": 65.05,
+      "completions/mean_terminated_length": 56.03928680419922,
+      "completions/min_length": 16.6,
+      "completions/min_terminated_length": 16.6,
+      "entropy": 0.13224927680566906,
+      "epoch": 15.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.1875,
+      "learning_rate": 4.958333333333334e-06,
+      "loss": -0.003383058309555054,
+      "num_tokens": 81545.0,
+      "reward": 0.3994179755449295,
+      "reward_std": 0.5622525057464373,
+      "rewards/reward_fn/mean": 0.3994179755449295,
+      "rewards/reward_fn/std": 0.56225254482124,
       "step": 30,
-      "step_time": 19.134268762199827
+      "step_time": 29.112151033400732
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -177,26 +177,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.1,
-      "completions/max_length": 150.8,
-      "completions/max_terminated_length": 120.2,
-      "completions/mean_length": 73.375,
-      "completions/mean_terminated_length": 54.36666717529297,
-      "completions/min_length": 15.2,
-      "completions/min_terminated_length": 15.2,
-      "entropy": 0.23285924410447478,
-      "epoch": 0.0015086206896551724,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.58203125,
-      "learning_rate": 4.879310344827586e-06,
-      "loss": -0.05149807929992676,
-      "num_tokens": 79734.0,
-      "reward": 0.46414998471736907,
-      "reward_std": 0.4071369742392562,
-      "rewards/reward_fn/mean": 0.46414998471736907,
-      "rewards/reward_fn/std": 0.40713699695188554,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 131.0,
+      "completions/max_terminated_length": 128.6,
+      "completions/mean_length": 61.075,
+      "completions/mean_terminated_length": 53.282144165039064,
+      "completions/min_length": 17.6,
+      "completions/min_terminated_length": 17.6,
+      "entropy": 0.11974610288161784,
+      "epoch": 17.5,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.220703125,
+      "learning_rate": 4.935185185185186e-06,
+      "loss": 0.004663025587797165,
+      "num_tokens": 93892.0,
+      "reward": 0.3469119846820831,
+      "reward_std": 0.5775202971824911,
+      "rewards/reward_fn/mean": 0.3469119846820831,
+      "rewards/reward_fn/std": 0.5775203009106917,
       "step": 35,
-      "step_time": 24.679699858600042
+      "step_time": 22.457519923200017
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -204,26 +204,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.4,
-      "completions/max_terminated_length": 125.4,
-      "completions/mean_length": 68.925,
-      "completions/mean_terminated_length": 68.925,
-      "completions/min_length": 26.0,
-      "completions/min_terminated_length": 26.0,
-      "entropy": 0.3110098702833056,
-      "epoch": 0.0017241379310344827,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.419921875,
-      "learning_rate": 4.836206896551724e-06,
-      "loss": 0.030162644386291505,
-      "num_tokens": 91695.0,
-      "reward": 0.5143539935350419,
-      "reward_std": 0.2790891878306866,
-      "rewards/reward_fn/mean": 0.5143539935350419,
-      "rewards/reward_fn/std": 0.279089218378067,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 179.6,
+      "completions/max_terminated_length": 131.2,
+      "completions/mean_length": 78.025,
+      "completions/mean_terminated_length": 65.70357513427734,
+      "completions/min_length": 19.0,
+      "completions/min_terminated_length": 19.0,
+      "entropy": 0.12550681543070824,
+      "epoch": 20.0,
+      "frac_reward_zero_std": 0.4,
+      "grad_norm": 0.4609375,
+      "learning_rate": 4.9120370370370375e-06,
+      "loss": 0.036792796850204465,
+      "num_tokens": 107209.0,
+      "reward": 0.3498039901256561,
+      "reward_std": 0.7408117946935817,
+      "rewards/reward_fn/mean": 0.3498039901256561,
+      "rewards/reward_fn/std": 0.7408118456369266,
       "step": 40,
-      "step_time": 21.565513409599998
+      "step_time": 28.692756705999635
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -231,26 +231,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 100.0,
-      "completions/max_terminated_length": 100.0,
-      "completions/mean_length": 53.875,
-      "completions/mean_terminated_length": 53.875,
-      "completions/min_length": 16.6,
-      "completions/min_terminated_length": 16.6,
-      "entropy": 0.23047098610550165,
-      "epoch": 0.001939655172413793,
-      "frac_reward_zero_std": 0.4,
-      "grad_norm": 1.15625,
-      "learning_rate": 4.793103448275862e-06,
-      "loss": -0.04350074529647827,
-      "num_tokens": 102994.0,
-      "reward": 0.6398920059204102,
-      "reward_std": 0.2279826147481799,
-      "rewards/reward_fn/mean": 0.6398920059204102,
-      "rewards/reward_fn/std": 0.22798261437565087,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 189.0,
+      "completions/max_terminated_length": 178.8,
+      "completions/mean_length": 92.675,
+      "completions/mean_terminated_length": 89.51071472167969,
+      "completions/min_length": 17.8,
+      "completions/min_terminated_length": 17.8,
+      "entropy": 0.15165529411751777,
+      "epoch": 22.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.232421875,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": -0.004727205634117127,
+      "num_tokens": 120868.0,
+      "reward": 0.30072798430919645,
+      "reward_std": 0.7614144545921591,
+      "rewards/reward_fn/mean": 0.30072798430919645,
+      "rewards/reward_fn/std": 0.7614145380415721,
       "step": 45,
-      "step_time": 18.23975218899968
+      "step_time": 30.053675796201425
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -258,26 +258,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 133.4,
-      "completions/max_terminated_length": 133.4,
-      "completions/mean_length": 75.975,
-      "completions/mean_terminated_length": 75.975,
-      "completions/min_length": 23.8,
-      "completions/min_terminated_length": 23.8,
-      "entropy": 0.34613882582634686,
-      "epoch": 0.0021551724137931034,
-      "frac_reward_zero_std": 0.2,
-      "grad_norm": 0.1318359375,
-      "learning_rate": 4.75e-06,
-      "loss": -0.03668657541275024,
-      "num_tokens": 115289.0,
-      "reward": 0.33042599707841874,
-      "reward_std": 0.5918701648712158,
-      "rewards/reward_fn/mean": 0.33042599707841874,
-      "rewards/reward_fn/std": 0.5918702006340026,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 208.6,
+      "completions/max_terminated_length": 189.6,
+      "completions/mean_length": 102.625,
+      "completions/mean_terminated_length": 90.11666870117188,
+      "completions/min_length": 18.8,
+      "completions/min_terminated_length": 18.8,
+      "entropy": 0.15195838457439095,
+      "epoch": 25.0,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.208984375,
+      "learning_rate": 4.865740740740741e-06,
+      "loss": 0.04500017166137695,
+      "num_tokens": 135121.0,
+      "reward": 0.5009119868278503,
+      "reward_std": 0.5213470441231038,
+      "rewards/reward_fn/mean": 0.5009119868278503,
+      "rewards/reward_fn/std": 0.5213470560469432,
       "step": 50,
-      "step_time": 22.43372436519985
+      "step_time": 32.55890014459801
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -285,26 +285,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.2,
-      "completions/max_terminated_length": 92.2,
-      "completions/mean_length": 50.65,
-      "completions/mean_terminated_length": 50.65,
-      "completions/min_length": 18.0,
-      "completions/min_terminated_length": 18.0,
-      "entropy": 0.3391244841972366,
-      "epoch": 0.0023706896551724138,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.39453125,
-      "learning_rate": 4.706896551724138e-06,
-      "loss": -0.0521969735622406,
-      "num_tokens": 126111.0,
-      "reward": 0.6139639973640442,
-      "reward_std": 0.18208687230944634,
-      "rewards/reward_fn/mean": 0.6139639973640442,
-      "rewards/reward_fn/std": 0.18208688013255597,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 195.4,
+      "completions/max_terminated_length": 188.4,
+      "completions/mean_length": 110.95,
+      "completions/mean_terminated_length": 106.75357208251953,
+      "completions/min_length": 51.8,
+      "completions/min_terminated_length": 51.8,
+      "entropy": 0.1545537636615336,
+      "epoch": 27.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 4.842592592592593e-06,
+      "loss": 0.04118683934211731,
+      "num_tokens": 149755.0,
+      "reward": 0.40073599219322203,
+      "reward_std": 0.6958227735885885,
+      "rewards/reward_fn/mean": 0.40073599219322203,
+      "rewards/reward_fn/std": 0.6958228093542858,
       "step": 55,
-      "step_time": 16.889007100199933
+      "step_time": 30.86228356460051
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -312,26 +312,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.025,
-      "completions/max_length": 164.0,
-      "completions/max_terminated_length": 162.4,
-      "completions/mean_length": 82.1,
-      "completions/mean_terminated_length": 78.87857208251953,
-      "completions/min_length": 29.4,
-      "completions/min_terminated_length": 29.4,
-      "entropy": 0.28240502553526314,
-      "epoch": 0.002586206896551724,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 172.8,
+      "completions/max_terminated_length": 147.2,
+      "completions/mean_length": 91.875,
+      "completions/mean_terminated_length": 81.20119171142578,
+      "completions/min_length": 32.8,
+      "completions/min_terminated_length": 32.8,
+      "entropy": 0.14326929268427194,
+      "epoch": 30.0,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.2373046875,
-      "learning_rate": 4.663793103448276e-06,
-      "loss": -0.0108717679977417,
-      "num_tokens": 138667.0,
-      "reward": 0.6616780042648316,
-      "reward_std": 0.18088504523038865,
-      "rewards/reward_fn/mean": 0.6616780042648316,
-      "rewards/reward_fn/std": 0.18088504672050476,
+      "grad_norm": 0.318359375,
+      "learning_rate": 4.819444444444445e-06,
+      "loss": 0.06774483323097229,
+      "num_tokens": 163334.0,
+      "reward": 0.5998779892921448,
+      "reward_std": 0.3705297726322897,
+      "rewards/reward_fn/mean": 0.5998779892921448,
+      "rewards/reward_fn/std": 0.3705297749955207,
       "step": 60,
-      "step_time": 26.46498533039994
+      "step_time": 27.902300025800287
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -339,26 +339,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.05,
-      "completions/max_length": 134.8,
-      "completions/max_terminated_length": 133.0,
-      "completions/mean_length": 69.55,
-      "completions/mean_terminated_length": 63.233334350585935,
-      "completions/min_length": 15.8,
-      "completions/min_terminated_length": 15.8,
-      "entropy": 0.3688268234953284,
-      "epoch": 0.0028017241379310344,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.2216796875,
-      "learning_rate": 4.620689655172414e-06,
-      "loss": 0.09875075221061706,
-      "num_tokens": 150585.0,
-      "reward": 0.5008320093154908,
-      "reward_std": 0.2221744753420353,
-      "rewards/reward_fn/mean": 0.5008320093154908,
-      "rewards/reward_fn/std": 0.22217447832226753,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 152.4,
+      "completions/max_terminated_length": 147.6,
+      "completions/mean_length": 81.625,
+      "completions/mean_terminated_length": 78.91071472167968,
+      "completions/min_length": 33.0,
+      "completions/min_terminated_length": 33.0,
+      "entropy": 0.15796504009049386,
+      "epoch": 32.5,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.0,
+      "learning_rate": 4.796296296296297e-06,
+      "loss": -0.008640621602535248,
+      "num_tokens": 176503.0,
+      "reward": 0.45116598904132843,
+      "reward_std": 0.4201629768765997,
+      "rewards/reward_fn/mean": 0.45116598904132843,
+      "rewards/reward_fn/std": 0.42016300329414663,
       "step": 65,
-      "step_time": 22.69940989719953
+      "step_time": 25.321657606401278
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -367,25 +367,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.05,
-      "completions/max_length": 149.4,
-      "completions/max_terminated_length": 141.4,
-      "completions/mean_length": 82.125,
-      "completions/mean_terminated_length": 76.2107162475586,
-      "completions/min_length": 34.8,
-      "completions/min_terminated_length": 34.8,
-      "entropy": 0.2419425747357309,
-      "epoch": 0.003017241379310345,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.5859375,
-      "learning_rate": 4.577586206896552e-06,
-      "loss": -0.06608393192291259,
-      "num_tokens": 162726.0,
-      "reward": 0.35277000069618225,
-      "reward_std": 0.37774557205848397,
-      "rewards/reward_fn/mean": 0.35277000069618225,
-      "rewards/reward_fn/std": 0.37774557354860006,
+      "completions/max_length": 197.6,
+      "completions/max_terminated_length": 183.2,
+      "completions/mean_length": 114.3,
+      "completions/mean_terminated_length": 109.30833435058594,
+      "completions/min_length": 50.0,
+      "completions/min_terminated_length": 50.0,
+      "entropy": 0.16214836928993465,
+      "epoch": 35.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.0,
+      "learning_rate": 4.7731481481481484e-06,
+      "loss": 0.007857038080692292,
+      "num_tokens": 191271.0,
+      "reward": 0.29932799339294436,
+      "reward_std": 0.6332858696579933,
+      "rewards/reward_fn/mean": 0.29932799339294436,
+      "rewards/reward_fn/std": 0.6332858689129353,
       "step": 70,
-      "step_time": 24.30778650340026
+      "step_time": 31.062436907000667
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -393,26 +393,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.025,
-      "completions/max_length": 137.6,
-      "completions/max_terminated_length": 135.6,
-      "completions/mean_length": 77.55,
-      "completions/mean_terminated_length": 74.46785888671874,
-      "completions/min_length": 41.8,
-      "completions/min_terminated_length": 41.8,
-      "entropy": 0.35793030727654696,
-      "epoch": 0.003232758620689655,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.63671875,
-      "learning_rate": 4.53448275862069e-06,
-      "loss": 0.10921311378479004,
-      "num_tokens": 175364.0,
-      "reward": 0.15934600830078124,
-      "reward_std": 0.5374872148036957,
-      "rewards/reward_fn/mean": 0.15934600830078124,
-      "rewards/reward_fn/std": 0.5374872386455536,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 179.4,
+      "completions/max_terminated_length": 173.2,
+      "completions/mean_length": 81.875,
+      "completions/mean_terminated_length": 72.65,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "entropy": 0.1165547446347773,
+      "epoch": 37.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.4609375,
+      "learning_rate": 4.75e-06,
+      "loss": 0.015596307814121246,
+      "num_tokens": 204450.0,
+      "reward": 0.5023359954357147,
+      "reward_std": 0.39528531452669996,
+      "rewards/reward_fn/mean": 0.5023359954357147,
+      "rewards/reward_fn/std": 0.3952853078444605,
       "step": 75,
-      "step_time": 23.099894465200123
+      "step_time": 28.755046762000347
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -420,26 +420,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.6,
-      "completions/max_terminated_length": 95.6,
-      "completions/mean_length": 53.175,
-      "completions/mean_terminated_length": 53.175,
-      "completions/min_length": 15.8,
-      "completions/min_terminated_length": 15.8,
-      "entropy": 0.2758270605234429,
-      "epoch": 0.0034482758620689655,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.443359375,
-      "learning_rate": 4.4913793103448275e-06,
-      "loss": 0.11924625635147094,
-      "num_tokens": 186407.0,
-      "reward": 0.6774819850921631,
-      "reward_std": 0.11054287778679281,
-      "rewards/reward_fn/mean": 0.6774819850921631,
-      "rewards/reward_fn/std": 0.11054288879968226,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 204.4,
+      "completions/max_terminated_length": 181.4,
+      "completions/mean_length": 111.05,
+      "completions/mean_terminated_length": 104.30357208251954,
+      "completions/min_length": 44.6,
+      "completions/min_terminated_length": 44.6,
+      "entropy": 0.13618089363444597,
+      "epoch": 40.0,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.140625,
+      "learning_rate": 4.726851851851852e-06,
+      "loss": 0.043748298287391664,
+      "num_tokens": 219088.0,
+      "reward": 0.3999840050935745,
+      "reward_std": 0.5622952695033746,
+      "rewards/reward_fn/mean": 0.3999840050935745,
+      "rewards/reward_fn/std": 0.5622952873265603,
       "step": 80,
-      "step_time": 17.348075130399685
+      "step_time": 31.989889109399883
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -447,26 +447,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 137.6,
-      "completions/max_terminated_length": 137.6,
-      "completions/mean_length": 68.25,
-      "completions/mean_terminated_length": 68.25,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "entropy": 0.2696234828326851,
-      "epoch": 0.003663793103448276,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.287109375,
-      "learning_rate": 4.4482758620689656e-06,
-      "loss": -0.03822631537914276,
-      "num_tokens": 198241.0,
-      "reward": 0.5305620029568672,
-      "reward_std": 0.4029154841089621,
-      "rewards/reward_fn/mean": 0.5305620029568672,
-      "rewards/reward_fn/std": 0.4029154877178371,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 191.4,
+      "completions/max_terminated_length": 186.4,
+      "completions/mean_length": 104.1,
+      "completions/mean_terminated_length": 100.05000152587891,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "entropy": 0.17853650995530188,
+      "epoch": 42.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.1875,
+      "learning_rate": 4.703703703703704e-06,
+      "loss": 0.02743053436279297,
+      "num_tokens": 233420.0,
+      "reward": 0.44984598755836486,
+      "reward_std": 0.5406898662215098,
+      "rewards/reward_fn/mean": 0.44984598755836486,
+      "rewards/reward_fn/std": 0.5406898936373181,
       "step": 85,
-      "step_time": 22.834117503599828
+      "step_time": 30.449153596599718
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -474,26 +474,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 131.2,
-      "completions/max_terminated_length": 131.2,
-      "completions/mean_length": 77.45,
-      "completions/mean_terminated_length": 77.45,
-      "completions/min_length": 19.8,
-      "completions/min_terminated_length": 19.8,
-      "entropy": 0.3707127865403891,
-      "epoch": 0.003879310344827586,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.50390625,
-      "learning_rate": 4.405172413793104e-06,
-      "loss": 0.035861659049987796,
-      "num_tokens": 210935.0,
-      "reward": 0.3616240084171295,
-      "reward_std": 0.48762375079095366,
-      "rewards/reward_fn/mean": 0.3616240084171295,
-      "rewards/reward_fn/std": 0.48762374445796014,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 200.6,
+      "completions/max_terminated_length": 186.6,
+      "completions/mean_length": 103.875,
+      "completions/mean_terminated_length": 88.48499984741211,
+      "completions/min_length": 31.2,
+      "completions/min_terminated_length": 31.2,
+      "entropy": 0.12900277911685407,
+      "epoch": 45.0,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.2314453125,
+      "learning_rate": 4.680555555555556e-06,
+      "loss": 0.05115030407905578,
+      "num_tokens": 247507.0,
+      "reward": 0.5499680012464523,
+      "reward_std": 0.39219149924028895,
+      "rewards/reward_fn/mean": 0.5499680012464523,
+      "rewards/reward_fn/std": 0.3921915319937398,
       "step": 90,
-      "step_time": 22.48314213299982
+      "step_time": 31.51559891059951
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -501,26 +501,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.075,
-      "completions/max_length": 118.2,
-      "completions/max_terminated_length": 116.0,
-      "completions/mean_length": 66.9,
-      "completions/mean_terminated_length": 56.7,
-      "completions/min_length": 25.0,
-      "completions/min_terminated_length": 25.0,
-      "entropy": 0.342819757014513,
-      "epoch": 0.004094827586206897,
-      "frac_reward_zero_std": 0.2,
-      "grad_norm": 0.78125,
-      "learning_rate": 4.362068965517242e-06,
-      "loss": -0.051665693521499634,
-      "num_tokens": 222647.0,
-      "reward": 0.46663198471069334,
-      "reward_std": 0.3400567059754394,
-      "rewards/reward_fn/mean": 0.46663198471069334,
-      "rewards/reward_fn/std": 0.34005671155173334,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 180.8,
+      "completions/max_terminated_length": 167.4,
+      "completions/mean_length": 103.05,
+      "completions/mean_terminated_length": 99.05000152587891,
+      "completions/min_length": 44.6,
+      "completions/min_terminated_length": 44.6,
+      "entropy": 0.12843654905445873,
+      "epoch": 47.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.099609375,
+      "learning_rate": 4.6574074074074076e-06,
+      "loss": 0.043516650795936584,
+      "num_tokens": 261825.0,
+      "reward": 0.4511160016059875,
+      "reward_std": 0.6716319680213928,
+      "rewards/reward_fn/mean": 0.4511160016059875,
+      "rewards/reward_fn/std": 0.6716320037841796,
       "step": 95,
-      "step_time": 20.31996373819993
+      "step_time": 28.928432848399826
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -528,26 +528,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 82.2,
-      "completions/max_terminated_length": 82.2,
-      "completions/mean_length": 39.975,
-      "completions/mean_terminated_length": 39.975,
-      "completions/min_length": 16.4,
-      "completions/min_terminated_length": 16.4,
-      "entropy": 0.25842549465596676,
-      "epoch": 0.004310344827586207,
-      "frac_reward_zero_std": 0.2,
-      "grad_norm": 0.64453125,
-      "learning_rate": 4.31896551724138e-06,
-      "loss": 0.029389530420303345,
-      "num_tokens": 233466.0,
-      "reward": 0.5307899951934815,
-      "reward_std": 0.28180397795513273,
-      "rewards/reward_fn/mean": 0.5307899951934815,
-      "rewards/reward_fn/std": 0.28180398060940204,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 157.0,
+      "completions/max_terminated_length": 151.8,
+      "completions/mean_length": 84.425,
+      "completions/mean_terminated_length": 80.66428680419922,
+      "completions/min_length": 35.8,
+      "completions/min_terminated_length": 35.8,
+      "entropy": 0.13799221392255276,
+      "epoch": 50.0,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.154296875,
+      "learning_rate": 4.634259259259259e-06,
+      "loss": -0.0032827585935592653,
+      "num_tokens": 275106.0,
+      "reward": 0.5485600084066391,
+      "reward_std": 0.39775218120921635,
+      "rewards/reward_fn/mean": 0.5485600084066391,
+      "rewards/reward_fn/std": 0.3977522020417382,
       "step": 100,
-      "step_time": 15.753981888000453
+      "step_time": 25.882849212799556
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -555,26 +555,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.8,
-      "completions/max_terminated_length": 77.8,
-      "completions/mean_length": 49.6,
-      "completions/mean_terminated_length": 49.6,
-      "completions/min_length": 16.4,
-      "completions/min_terminated_length": 16.4,
-      "entropy": 0.27716706816572695,
-      "epoch": 0.004525862068965518,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 182.2,
+      "completions/max_terminated_length": 174.8,
+      "completions/mean_length": 100.925,
+      "completions/mean_terminated_length": 96.83928680419922,
+      "completions/min_length": 40.4,
+      "completions/min_terminated_length": 40.4,
+      "entropy": 0.12651287836488337,
+      "epoch": 52.5,
       "frac_reward_zero_std": 0.6,
-      "grad_norm": 0.51953125,
-      "learning_rate": 4.275862068965518e-06,
-      "loss": 0.011613726615905762,
-      "num_tokens": 244550.0,
-      "reward": 0.6918019950389862,
-      "reward_std": 0.10386521331965923,
-      "rewards/reward_fn/mean": 0.6918019950389862,
-      "rewards/reward_fn/std": 0.10386521825566888,
+      "grad_norm": 0.0,
+      "learning_rate": 4.611111111111112e-06,
+      "loss": -0.008606845140457153,
+      "num_tokens": 289075.0,
+      "reward": 0.5499760001897812,
+      "reward_std": 0.3922359466421767,
+      "rewards/reward_fn/mean": 0.5499760001897812,
+      "rewards/reward_fn/std": 0.39223596563824686,
       "step": 105,
-      "step_time": 15.171548829199946
+      "step_time": 29.0759426169996
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -582,26 +582,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 94.0,
-      "completions/max_terminated_length": 94.0,
-      "completions/mean_length": 47.1,
-      "completions/mean_terminated_length": 47.1,
-      "completions/min_length": 16.0,
-      "completions/min_terminated_length": 16.0,
-      "entropy": 0.1942274721484864,
-      "epoch": 0.0047413793103448275,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.515625,
-      "learning_rate": 4.232758620689656e-06,
-      "loss": -0.000987192615866661,
-      "num_tokens": 255962.0,
-      "reward": 0.1787539839744568,
-      "reward_std": 0.5241983281448483,
-      "rewards/reward_fn/mean": 0.1787539839744568,
-      "rewards/reward_fn/std": 0.5241983408108354,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 194.0,
+      "completions/max_terminated_length": 169.0,
+      "completions/mean_length": 104.525,
+      "completions/mean_terminated_length": 98.08214416503907,
+      "completions/min_length": 43.2,
+      "completions/min_terminated_length": 43.2,
+      "entropy": 0.16052292285021394,
+      "epoch": 55.0,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.0,
+      "learning_rate": 4.587962962962964e-06,
+      "loss": 0.022305211424827574,
+      "num_tokens": 303424.0,
+      "reward": 0.5999760001897811,
+      "reward_std": 0.34845718718279384,
+      "rewards/reward_fn/mean": 0.5999760001897811,
+      "rewards/reward_fn/std": 0.3484572199362447,
       "step": 110,
-      "step_time": 17.40271624299976
+      "step_time": 30.576252812399254
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -609,26 +609,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 108.4,
-      "completions/max_terminated_length": 108.4,
-      "completions/mean_length": 54.025,
-      "completions/mean_terminated_length": 54.025,
-      "completions/min_length": 26.8,
-      "completions/min_terminated_length": 26.8,
-      "entropy": 0.21391218816424953,
-      "epoch": 0.004956896551724138,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.546875,
-      "learning_rate": 4.189655172413794e-06,
-      "loss": -0.07643186450004577,
-      "num_tokens": 267563.0,
-      "reward": 0.5654179990291596,
-      "reward_std": 0.2869014423340559,
-      "rewards/reward_fn/mean": 0.5654179990291596,
-      "rewards/reward_fn/std": 0.2869014598429203,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 186.2,
+      "completions/max_terminated_length": 170.0,
+      "completions/mean_length": 107.275,
+      "completions/mean_terminated_length": 97.70833435058594,
+      "completions/min_length": 42.2,
+      "completions/min_terminated_length": 42.2,
+      "entropy": 0.16247000750154256,
+      "epoch": 57.5,
+      "frac_reward_zero_std": 0.9,
+      "grad_norm": 0.0,
+      "learning_rate": 4.564814814814815e-06,
+      "loss": 0.036947214603424074,
+      "num_tokens": 317863.0,
+      "reward": 0.700000011920929,
+      "reward_std": 0.1851640224456787,
+      "rewards/reward_fn/mean": 0.700000011920929,
+      "rewards/reward_fn/std": 0.1851640224456787,
       "step": 115,
-      "step_time": 19.255251389799923
+      "step_time": 29.630989668599433
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -636,26 +636,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.025,
-      "completions/max_length": 136.0,
-      "completions/max_terminated_length": 124.6,
-      "completions/mean_length": 80.575,
-      "completions/mean_terminated_length": 77.74285888671875,
-      "completions/min_length": 33.0,
-      "completions/min_terminated_length": 33.0,
-      "entropy": 0.31822405869606885,
-      "epoch": 0.005172413793103448,
-      "frac_reward_zero_std": 0.2,
-      "grad_norm": 1.2421875,
-      "learning_rate": 4.146551724137932e-06,
-      "loss": 0.01645863652229309,
-      "num_tokens": 280074.0,
-      "reward": 0.06659798473119735,
-      "reward_std": 0.5049000799655914,
-      "rewards/reward_fn/mean": 0.06659798473119735,
-      "rewards/reward_fn/std": 0.5049000844359398,
+      "completions/clipped_ratio": 0.05,
+      "completions/max_length": 202.0,
+      "completions/max_terminated_length": 183.2,
+      "completions/mean_length": 106.1,
+      "completions/mean_terminated_length": 99.70714416503907,
+      "completions/min_length": 40.8,
+      "completions/min_terminated_length": 40.8,
+      "entropy": 0.15718324964400382,
+      "epoch": 60.0,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.0,
+      "learning_rate": 4.541666666666667e-06,
+      "loss": 0.04491000473499298,
+      "num_tokens": 332059.0,
+      "reward": 0.2999920010566711,
+      "reward_std": 0.6278709468911984,
+      "rewards/reward_fn/mean": 0.2999920010566711,
+      "rewards/reward_fn/std": 0.6278709915655781,
       "step": 120,
-      "step_time": 22.649546912000005
+      "step_time": 31.635853910601874
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -663,26 +663,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.05,
-      "completions/max_length": 138.4,
-      "completions/max_terminated_length": 136.2,
-      "completions/mean_length": 68.85,
-      "completions/mean_terminated_length": 61.975,
-      "completions/min_length": 25.4,
-      "completions/min_terminated_length": 25.4,
-      "entropy": 0.33708888398250564,
-      "epoch": 0.005387931034482759,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 1.25,
-      "learning_rate": 4.103448275862069e-06,
-      "loss": 0.16269092559814452,
-      "num_tokens": 291760.0,
-      "reward": 0.09610399603843689,
-      "reward_std": 0.6181229472160339,
-      "rewards/reward_fn/mean": 0.09610399603843689,
-      "rewards/reward_fn/std": 0.6181229710578918,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 170.6,
+      "completions/max_terminated_length": 170.6,
+      "completions/mean_length": 94.575,
+      "completions/mean_terminated_length": 94.575,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
+      "entropy": 0.10850867056287825,
+      "epoch": 62.5,
+      "frac_reward_zero_std": 0.8,
+      "grad_norm": 0.0,
+      "learning_rate": 4.5185185185185185e-06,
+      "loss": 0.024337136745452882,
+      "num_tokens": 345990.0,
+      "reward": 0.6500000119209289,
+      "reward_std": 0.3265853762626648,
+      "rewards/reward_fn/mean": 0.6500000119209289,
+      "rewards/reward_fn/std": 0.3265853762626648,
       "step": 125,
-      "step_time": 22.892032967400247
+      "step_time": 27.550983037801416
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -690,26 +690,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.05,
-      "completions/max_length": 121.6,
-      "completions/max_terminated_length": 119.6,
-      "completions/mean_length": 54.625,
-      "completions/mean_terminated_length": 45.516667175292966,
-      "completions/min_length": 15.4,
-      "completions/min_terminated_length": 15.4,
-      "entropy": 0.2316889252353576,
-      "epoch": 0.005603448275862069,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.78515625,
-      "learning_rate": 4.060344827586207e-06,
-      "loss": 0.05887578129768371,
-      "num_tokens": 302777.0,
-      "reward": 0.47827199250459673,
-      "reward_std": 0.31257240688428284,
-      "rewards/reward_fn/mean": 0.47827199250459673,
-      "rewards/reward_fn/std": 0.31257240623235705,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 201.8,
+      "completions/max_terminated_length": 187.0,
+      "completions/mean_length": 99.25,
+      "completions/mean_terminated_length": 84.59642944335937,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "entropy": 0.12130871089175344,
+      "epoch": 65.0,
+      "frac_reward_zero_std": 0.8,
+      "grad_norm": 0.0,
+      "learning_rate": 4.49537037037037e-06,
+      "loss": 0.018311865627765656,
+      "num_tokens": 359912.0,
+      "reward": 0.6000000089406967,
+      "reward_std": 0.34844101667404176,
+      "rewards/reward_fn/mean": 0.6000000089406967,
+      "rewards/reward_fn/std": 0.34844104051589964,
       "step": 130,
-      "step_time": 20.756939186200178
+      "step_time": 31.70451795959889
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -718,25 +718,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.4,
-      "completions/max_terminated_length": 83.4,
-      "completions/mean_length": 43.375,
-      "completions/mean_terminated_length": 43.375,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
-      "entropy": 0.14220066292909905,
-      "epoch": 0.00581896551724138,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.361328125,
-      "learning_rate": 4.017241379310346e-06,
-      "loss": 0.0067549549043178555,
-      "num_tokens": 313896.0,
-      "reward": 0.5540139883756637,
-      "reward_std": 0.3137821853160858,
-      "rewards/reward_fn/mean": 0.5540139883756637,
-      "rewards/reward_fn/std": 0.31378220971673726,
+      "completions/max_length": 162.6,
+      "completions/max_terminated_length": 162.6,
+      "completions/mean_length": 89.825,
+      "completions/mean_terminated_length": 89.825,
+      "completions/min_length": 43.0,
+      "completions/min_terminated_length": 43.0,
+      "entropy": 0.11614590694662183,
+      "epoch": 67.5,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 4.472222222222223e-06,
+      "loss": 0.019238683581352233,
+      "num_tokens": 373457.0,
+      "reward": 0.5999920129776001,
+      "reward_std": 0.3703506765436032,
+      "rewards/reward_fn/mean": 0.5999920129776001,
+      "rewards/reward_fn/std": 0.3703506735342671,
       "step": 135,
-      "step_time": 16.04840490619972
+      "step_time": 26.545528039799684
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.2,
-      "completions/max_terminated_length": 65.2,
-      "completions/mean_length": 33.825,
-      "completions/mean_terminated_length": 33.825,
-      "completions/min_length": 15.8,
-      "completions/min_terminated_length": 15.8,
-      "entropy": 0.16708708838559688,
-      "epoch": 0.00603448275862069,
-      "frac_reward_zero_std": 0.3,
-      "grad_norm": 0.53125,
-      "learning_rate": 3.974137931034483e-06,
-      "loss": 0.000922510027885437,
-      "num_tokens": 324093.0,
-      "reward": 0.5665820002555847,
-      "reward_std": 0.3455619063111953,
-      "rewards/reward_fn/mean": 0.5665820002555847,
-      "rewards/reward_fn/std": 0.34556191868614405,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 205.4,
+      "completions/max_terminated_length": 165.4,
+      "completions/mean_length": 99.4,
+      "completions/mean_terminated_length": 87.00476379394532,
+      "completions/min_length": 19.4,
+      "completions/min_terminated_length": 19.4,
+      "entropy": 0.1428308295784518,
+      "epoch": 70.0,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.0,
+      "learning_rate": 4.449074074074075e-06,
+      "loss": 0.04644445776939392,
+      "num_tokens": 387581.0,
+      "reward": 0.5499920099973679,
+      "reward_std": 0.39217875003814695,
+      "rewards/reward_fn/mean": 0.5499920099973679,
+      "rewards/reward_fn/std": 0.39217878580093385,
       "step": 140,
-      "step_time": 13.251863584200146
+      "step_time": 32.013767499400274
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -771,26 +771,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.1,
-      "completions/max_length": 175.8,
-      "completions/max_terminated_length": 149.4,
-      "completions/mean_length": 76.9,
-      "completions/mean_terminated_length": 58.73142929077149,
-      "completions/min_length": 25.6,
-      "completions/min_terminated_length": 25.6,
-      "entropy": 0.2752605409361422,
-      "epoch": 0.00625,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.62109375,
-      "learning_rate": 3.931034482758621e-06,
-      "loss": -0.012939244508743286,
-      "num_tokens": 336493.0,
-      "reward": 0.03320800065994263,
-      "reward_std": 0.6782135218381882,
-      "rewards/reward_fn/mean": 0.03320800065994263,
-      "rewards/reward_fn/std": 0.6782135486602783,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 171.8,
+      "completions/max_terminated_length": 171.8,
+      "completions/mean_length": 97.65,
+      "completions/mean_terminated_length": 97.65,
+      "completions/min_length": 28.8,
+      "completions/min_terminated_length": 28.8,
+      "entropy": 0.13889142961706966,
+      "epoch": 72.5,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 4.425925925925927e-06,
+      "loss": 0.021968120336532594,
+      "num_tokens": 401439.0,
+      "reward": 0.6477920055389405,
+      "reward_std": 0.4305092230439186,
+      "rewards/reward_fn/mean": 0.6477920055389405,
+      "rewards/reward_fn/std": 0.4305092342197895,
       "step": 145,
-      "step_time": 27.717435833599847
+      "step_time": 27.826696390995494
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -799,31 +799,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.05,
-      "completions/max_length": 159.6,
-      "completions/max_terminated_length": 159.4,
-      "completions/mean_length": 86.4,
-      "completions/mean_terminated_length": 78.2,
-      "completions/min_length": 27.0,
-      "completions/min_terminated_length": 27.0,
-      "entropy": 0.3023837681859732,
-      "epoch": 0.00646551724137931,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 0.25,
-      "learning_rate": 3.887931034482759e-06,
-      "loss": -0.004035864770412445,
-      "num_tokens": 349213.0,
-      "reward": 0.2794659972190857,
-      "reward_std": 0.33909483794122935,
-      "rewards/reward_fn/mean": 0.2794659972190857,
-      "rewards/reward_fn/std": 0.3390948371961713,
+      "completions/max_length": 211.0,
+      "completions/max_terminated_length": 199.4,
+      "completions/mean_length": 110.125,
+      "completions/mean_terminated_length": 102.15714416503906,
+      "completions/min_length": 36.6,
+      "completions/min_terminated_length": 36.6,
+      "entropy": 0.17309838379733264,
+      "epoch": 75.0,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.216796875,
+      "learning_rate": 4.4027777777777784e-06,
+      "loss": 0.0743155837059021,
+      "num_tokens": 415992.0,
+      "reward": 0.6007840156555175,
+      "reward_std": 0.3688644051551819,
+      "rewards/reward_fn/mean": 0.6007840156555175,
+      "rewards/reward_fn/std": 0.3688644051551819,
       "step": 150,
-      "step_time": 25.6254667358
+      "step_time": 32.79056119860106
     }
   ],
   "logging_steps": 5,
-  "max_steps": 600,
-  "num_input_tokens_seen": 349213,
-  "num_train_epochs": 1,
+  "max_steps": 1100,
+  "num_input_tokens_seen": 415992,
+  "num_train_epochs": 550,
   "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {

checkpoint-150/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cf50531ca359d39fbfa05dc0896e03532a83d53b029182b6d7f757efab0c97a
+oid sha256:354b92cb3fc579afc26aea1b098796649abd9be8f2b683beb58a14a6bab0c7d4
 size 7185