diff --git a/adapter_model.safetensors b/adapter_model.safetensors
index 91858393ce9c2d5cb7d204530d11113af62ab24d..0bee0aed05d4191a3acdedb0dc86170d1654847c 100644
--- a/adapter_model.safetensors
+++ b/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:37d50e9249dbc53f60027557052a860b27022847dd92972fcb41cbf4f17869a3
+oid sha256:ebdd83e68b5fdd48ae1bba30bc5d3100fe03446c96aea23fa9b9f3e1083e446a
size 8731128
diff --git a/checkpoint-100/adapter_model.safetensors b/checkpoint-100/adapter_model.safetensors
index a3cb46a185ca253ce73eaf9de0a5a55e368a1c8b..db066bb03e4d8a1bbc12effd711bb175c4bcfede 100644
--- a/checkpoint-100/adapter_model.safetensors
+++ b/checkpoint-100/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b765e914dcf9c550a8e56ee013a86678674612688bd65ab80410d7b255546e2f
+oid sha256:c00aef212698f881e6ff2200e43a5f91250d42926e6fbd47218b21500be5af7f
size 8731128
diff --git a/checkpoint-100/optimizer.pt b/checkpoint-100/optimizer.pt
index 2529ee600e59591f8284011977f830f44af25228..5505084ebc0fd6f3bf96abffc2a1c5ff36edbfab 100644
--- a/checkpoint-100/optimizer.pt
+++ b/checkpoint-100/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:a12e35a712271cd52af220515dc54813cb15987e84f037d958d20cd6cee68fff
+oid sha256:838026ac41f4ac6ae9a67d27339ddb6383efe04e56567e79aa9e24251055f37e
size 17526842
diff --git a/checkpoint-100/ref/adapter_model.safetensors b/checkpoint-100/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-100/ref/adapter_model.safetensors
+++ b/checkpoint-100/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-100/rng_state.pth b/checkpoint-100/rng_state.pth
index 1f81d453bd6b568764b658f5df2c4f2655db5d58..23ee1e6645585e171f70f5a91af3ac3aedbf8fc0 100644
--- a/checkpoint-100/rng_state.pth
+++ b/checkpoint-100/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:8ab604215494ab766d54b5d034814a38a5a131b983552e085e6ba89a07fe4f5b
+oid sha256:d48ef43f53ee2a844cfe6fb15c5f921d15a1deae9ef20e365f1b97ae85dc3e5e
size 14244
diff --git a/checkpoint-100/scheduler.pt b/checkpoint-100/scheduler.pt
index ba43507aebfbeb306a5d19f9fb9d5fc27a23cf0b..4763fcd9d5d64f91a2958b2a24acf6e920e62e9a 100644
--- a/checkpoint-100/scheduler.pt
+++ b/checkpoint-100/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:8c869bfb8f14e59279afcb8a8eecd370b83ad9128cc999745f22a1b121cdf645
+oid sha256:525284629670376fca25695ce8c77b361026259ca9754ab77c6eabb5d0d1e917
size 1064
diff --git a/checkpoint-100/trainer_state.json b/checkpoint-100/trainer_state.json
index 9732e82e7fcc4032ecac72aa0a54f4c05bd6f411..5954db5b6ab5f10c818ec95612bd71b66b3cb566 100644
--- a/checkpoint-100/trainer_state.json
+++ b/checkpoint-100/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 0.8130081300813008,
+ "epoch": 2.4390243902439024,
"eval_steps": 500,
"global_step": 100,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,32 +2788,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 411488,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 2458407,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -2828,7 +2828,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-100/training_args.bin b/checkpoint-100/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-100/training_args.bin
+++ b/checkpoint-100/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-150/adapter_model.safetensors b/checkpoint-150/adapter_model.safetensors
index a1502e3904dec405908eea76528099da99eb542a..82a830667ed74ce3b85aaed4be7f33746eb72057 100644
--- a/checkpoint-150/adapter_model.safetensors
+++ b/checkpoint-150/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:7c4e9a9589a7474f445663b6ff20677982b29ec3c7c1e0aaaf1ee48fbfad4ad5
+oid sha256:3b84fdd298768a6449636702b456e5d73a95a341f391abc7cce6fbfb2d5797a8
size 8731128
diff --git a/checkpoint-150/optimizer.pt b/checkpoint-150/optimizer.pt
index 7a16aa2d6d287e87c5ec87e9d597e417180bd9b1..aa1fe27938ceadef93b7066f05e3c33fbd948bfb 100644
--- a/checkpoint-150/optimizer.pt
+++ b/checkpoint-150/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d6beaea033a70db5b5a9402e7fcb22cd071216c551154452086b4d9eccaafd53
+oid sha256:763a96c5c84beb8f406445a3a15e9a28814cc76cfc83e4ba4438602eb95510b0
size 17526842
diff --git a/checkpoint-150/ref/adapter_model.safetensors b/checkpoint-150/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-150/ref/adapter_model.safetensors
+++ b/checkpoint-150/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-150/rng_state.pth b/checkpoint-150/rng_state.pth
index 759cb39a66adc9b479d54948cfa177fe6218bf5d..c2f0ea8506ce9f24695a25962a94fa6ce53cbb4c 100644
--- a/checkpoint-150/rng_state.pth
+++ b/checkpoint-150/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:27656c488218789eb6a077238f1346863125281337293d34eaf84a1aba2638e8
+oid sha256:8930fe80d4cf8b6fac8695e275ccbb0603f03c0586a874404f5b007a46b135e1
size 14244
diff --git a/checkpoint-150/scheduler.pt b/checkpoint-150/scheduler.pt
index e9e395f2702dc71dbebfa2593df822fb1fcd96a9..ba268722b762594c08a2c4f209a5dd252e653b25 100644
--- a/checkpoint-150/scheduler.pt
+++ b/checkpoint-150/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:ead0c69f231af9ff216a15725240624f552cc5e549122accf41fc9de1475d582
+oid sha256:d0614996c9ff4351e094e986310b25ed8b76a45f4e76585672a811323db08ea3
size 1064
diff --git a/checkpoint-150/trainer_state.json b/checkpoint-150/trainer_state.json
index 259c5cc6f9dea9547cbf25e4816fbe22c568375d..1b6c7a8ddcd22fa194ea44e0f55b4382e9be81cf 100644
--- a/checkpoint-150/trainer_state.json
+++ b/checkpoint-150/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 1.2195121951219512,
+ "epoch": 3.658536585365854,
"eval_steps": 500,
"global_step": 150,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
},
{
"clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.0,
- "completions/mean_terminated_length": 59.0,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2509461045265198,
- "epoch": 0.8211382113821138,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22887632250785828,
- "kl": 2.1612477212329395e-05,
- "learning_rate": 7.310023360571047e-07,
- "loss": 0.025605827569961548,
- "num_tokens": 414080.0,
- "reward": 0.588032603263855,
- "reward_std": 0.11032751202583313,
- "rewards/true_env_reward_fn/mean": 0.588032603263855,
- "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
"step": 101,
- "step_time": 3.625197022998691
+ "step_time": 16.478299477000064
},
{
"clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 176.0,
- "completions/max_terminated_length": 176.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5443179607391357,
- "epoch": 0.8292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10614532232284546,
- "kl": 1.492139062975184e-05,
- "learning_rate": 7.28862404677924e-07,
- "loss": 0.06531564146280289,
- "num_tokens": 419835.0,
- "reward": 0.07074306160211563,
- "reward_std": 0.2918013632297516,
- "rewards/true_env_reward_fn/mean": 0.07074306160211563,
- "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
"step": 102,
- "step_time": 7.796810614998321
+ "step_time": 11.344593008000174
},
{
"clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 101.0,
"completions/max_terminated_length": 101.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3223788738250732,
- "epoch": 0.8373983739837398,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2421368807554245,
- "kl": 3.265505938543356e-05,
- "learning_rate": 7.266930267388503e-07,
- "loss": -0.07752113044261932,
- "num_tokens": 422773.0,
- "reward": 0.33568501472473145,
- "reward_std": 0.2780380845069885,
- "rewards/true_env_reward_fn/mean": 0.33568501472473145,
- "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
"step": 103,
- "step_time": 4.313938073000827
+ "step_time": 8.135681302999728
},
{
"clip_ratio/high_max": 0.0,
@@ -2899,27 +2899,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 67.25,
- "completions/mean_terminated_length": 67.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.3332037329673767,
- "epoch": 0.8455284552845529,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13883370161056519,
- "kl": 2.1224042484391248e-05,
- "learning_rate": 7.244943964874369e-07,
- "loss": 0.021739646792411804,
- "num_tokens": 426507.0,
- "reward": 0.40595096349716187,
- "reward_std": 0.2035457342863083,
- "rewards/true_env_reward_fn/mean": 0.40595096349716187,
- "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
"step": 104,
- "step_time": 4.155937195999286
+ "step_time": 15.185034105999875
},
{
"clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
"completions/min_length": 31.0,
"completions/min_terminated_length": 31.0,
- "entropy": 1.3391229510307312,
- "epoch": 0.8536585365853658,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24205009639263153,
- "kl": 2.5022183763212524e-05,
- "learning_rate": 7.222667107905085e-07,
- "loss": 0.06330433487892151,
- "num_tokens": 429010.0,
- "reward": 0.3355163037776947,
- "reward_std": 0.2902730703353882,
- "rewards/true_env_reward_fn/mean": 0.3355163037776947,
- "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
"step": 105,
- "step_time": 3.808478789000219
+ "step_time": 9.588520330999927
},
{
"clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1864720582962036,
- "epoch": 0.8617886178861789,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13473568856716156,
- "kl": 1.4212585938366828e-05,
- "learning_rate": 7.200101691165338e-07,
- "loss": -0.020715661346912384,
- "num_tokens": 432403.0,
- "reward": 0.4871198534965515,
- "reward_std": 0.15407639741897583,
- "rewards/true_env_reward_fn/mean": 0.4871198534965515,
- "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
"step": 106,
- "step_time": 4.240638332001254
+ "step_time": 14.60399662399982
},
{
"clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.0,
- "completions/mean_terminated_length": 48.0,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "entropy": 1.0669284462928772,
- "epoch": 0.8699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14131899178028107,
- "kl": 1.5787159554747632e-05,
- "learning_rate": 7.177249735177651e-07,
- "loss": 0.03678784519433975,
- "num_tokens": 435995.0,
- "reward": 0.5010770559310913,
- "reward_std": 0.48966261744499207,
- "rewards/true_env_reward_fn/mean": 0.5010770559310913,
- "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
"step": 107,
- "step_time": 3.3587191269998584
+ "step_time": 20.100954443999854
},
{
"clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 71.875,
- "completions/mean_terminated_length": 71.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.304731547832489,
- "epoch": 0.8780487804878049,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.380985673284158e-05,
- "kl": 1.3128728824085556e-05,
- "learning_rate": 7.154113286121462e-07,
- "loss": 6.494262834166875e-07,
- "num_tokens": 442094.0,
- "reward": 0.4055500030517578,
- "reward_std": 0.052258480340242386,
- "rewards/true_env_reward_fn/mean": 0.4055500030517578,
- "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
"step": 108,
- "step_time": 4.337008413998774
+ "step_time": 8.304149297999857
},
{
"clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2324069738388062,
- "epoch": 0.8861788617886179,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13588950037956238,
- "kl": 1.3448377558233915e-05,
- "learning_rate": 7.130694415649912e-07,
- "loss": 1.0952353477478027e-06,
- "num_tokens": 447226.0,
- "reward": 0.20854972302913666,
- "reward_std": 0.06059705466032028,
- "rewards/true_env_reward_fn/mean": 0.20854972302913666,
- "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
"step": 109,
- "step_time": 3.2976038649994734
+ "step_time": 11.260021517000041
},
{
"clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1567262411117554,
- "epoch": 0.8943089430894309,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.736967720324174e-05,
- "kl": 1.2838129805459175e-05,
- "learning_rate": 7.106995220704342e-07,
- "loss": 6.425898391171359e-07,
- "num_tokens": 450359.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
"step": 110,
- "step_time": 4.067084037998939
+ "step_time": 22.80178854000019
},
{
"clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 100.0,
"completions/max_terminated_length": 100.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 12.0,
- "completions/min_terminated_length": 12.0,
- "entropy": 1.496058464050293,
- "epoch": 0.9024390243902439,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18041981756687164,
- "kl": 1.6616825632809196e-05,
- "learning_rate": 7.083017823326532e-07,
- "loss": 0.0269068144261837,
- "num_tokens": 453583.0,
- "reward": 0.5647265911102295,
- "reward_std": 0.1507105529308319,
- "rewards/true_env_reward_fn/mean": 0.5647265911102295,
- "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
"step": 111,
- "step_time": 4.347732382997492
+ "step_time": 9.663163859000178
},
{
"clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 67.75,
- "completions/mean_terminated_length": 67.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.4196155667304993,
- "epoch": 0.9105691056910569,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18451112508773804,
- "kl": 2.1803500203532167e-05,
- "learning_rate": 7.058764370468698e-07,
- "loss": 0.1650262475013733,
- "num_tokens": 456773.0,
- "reward": 0.6907394528388977,
- "reward_std": 0.1393815129995346,
- "rewards/true_env_reward_fn/mean": 0.6907394528388977,
- "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
"step": 112,
- "step_time": 4.627644968999448
+ "step_time": 9.903081222999617
},
{
"clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2243221998214722,
- "epoch": 0.9186991869918699,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331371545791626,
- "kl": 1.548633599668392e-05,
- "learning_rate": 7.034237033801247e-07,
- "loss": 0.039844345301389694,
- "num_tokens": 462714.0,
- "reward": 0.21676866710186005,
- "reward_std": 0.26559779047966003,
- "rewards/true_env_reward_fn/mean": 0.21676866710186005,
- "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
"step": 113,
- "step_time": 3.8455466220002563
+ "step_time": 10.101770388999284
},
{
"clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 73.0,
- "completions/mean_terminated_length": 73.0,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.325823724269867,
- "epoch": 0.926829268292683,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1866220384836197,
- "kl": 1.8801019905367866e-05,
- "learning_rate": 7.009438009518325e-07,
- "loss": 0.06504581868648529,
- "num_tokens": 465994.0,
- "reward": 0.5194582939147949,
- "reward_std": 0.2796703577041626,
- "rewards/true_env_reward_fn/mean": 0.5194582939147949,
- "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
"step": 114,
- "step_time": 4.151028698999653
+ "step_time": 17.633509853000305
},
{
"clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1521879434585571,
- "epoch": 0.9349593495934959,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14320029318332672,
- "kl": 1.2749982033710694e-05,
- "learning_rate": 6.98436951814117e-07,
- "loss": 0.03685300797224045,
- "num_tokens": 468615.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
"step": 115,
- "step_time": 3.6973990600017714
+ "step_time": 12.307247350000125
},
{
"clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.201507806777954,
- "epoch": 0.943089430894309,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14264807105064392,
- "kl": 2.6679515940486453e-05,
- "learning_rate": 6.959033804319283e-07,
- "loss": -0.023484818637371063,
- "num_tokens": 471647.0,
- "reward": 0.41836902499198914,
- "reward_std": 0.3116860091686249,
- "rewards/true_env_reward_fn/mean": 0.41836902499198914,
- "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
"step": 116,
- "step_time": 3.1295652919998247
+ "step_time": 14.468690254000194
},
{
"clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 62.375,
- "completions/mean_terminated_length": 62.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2834057807922363,
- "epoch": 0.9512195121951219,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.810227154754102e-05,
- "kl": 1.4841665233689127e-05,
- "learning_rate": 6.933433136629443e-07,
- "loss": 7.425555850204546e-07,
- "num_tokens": 474682.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
"step": 117,
- "step_time": 3.4368692790012574
+ "step_time": 11.849063975999798
},
{
"clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.191932499408722,
- "epoch": 0.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21204856038093567,
- "kl": 3.64198385796044e-05,
- "learning_rate": 6.907569807372574e-07,
- "loss": -0.001312553882598877,
- "num_tokens": 477027.0,
- "reward": 0.5300568342208862,
- "reward_std": 0.2945883274078369,
- "rewards/true_env_reward_fn/mean": 0.5300568342208862,
- "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
"step": 118,
- "step_time": 3.8569856240010267
+ "step_time": 16.597334930000216
},
{
"clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 190.0,
- "completions/max_terminated_length": 190.0,
- "completions/mean_length": 96.5,
- "completions/mean_terminated_length": 96.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2401175498962402,
- "epoch": 0.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011377666669432074,
- "kl": 1.3742283954343293e-05,
- "learning_rate": 6.881446132368494e-07,
- "loss": 6.866695230201003e-07,
- "num_tokens": 481999.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
"step": 119,
- "step_time": 8.09440958399864
+ "step_time": 11.729475523999554
},
{
"clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.230682611465454,
- "epoch": 0.975609756097561,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22175048291683197,
- "kl": 1.2522132237791084e-05,
- "learning_rate": 6.855064450748555e-07,
- "loss": -0.04083740711212158,
- "num_tokens": 490884.0,
- "reward": 0.13476666808128357,
- "reward_std": 0.2987530529499054,
- "rewards/true_env_reward_fn/mean": 0.13476666808128357,
- "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
"step": 120,
- "step_time": 4.678523641001448
+ "step_time": 11.503627788000358
},
{
"clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2799639105796814,
- "epoch": 0.983739837398374,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19926966726779938,
- "kl": 1.7022688552970067e-05,
- "learning_rate": 6.828427124746189e-07,
- "loss": -0.010804429650306702,
- "num_tokens": 496404.0,
- "reward": 0.24633333086967468,
- "reward_std": 0.2454334795475006,
- "rewards/true_env_reward_fn/mean": 0.24633333086967468,
- "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
"step": 121,
- "step_time": 3.98071062300005
+ "step_time": 8.728293746000418
},
{
"clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 46.875,
- "completions/mean_terminated_length": 46.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3840235471725464,
- "epoch": 0.991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24853822588920593,
- "kl": 3.688259130285587e-05,
- "learning_rate": 6.801536539485403e-07,
- "loss": 0.10205884277820587,
- "num_tokens": 499767.0,
- "reward": 0.3045905530452728,
- "reward_std": 0.262839138507843,
- "rewards/true_env_reward_fn/mean": 0.3045905530452728,
- "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
"step": 122,
- "step_time": 3.3792565210005705
+ "step_time": 15.51957702100026
},
{
"clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 50.375,
- "completions/mean_terminated_length": 50.375,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2064164280891418,
- "epoch": 1.0,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1365528553724289,
- "kl": 2.434901080050622e-05,
- "learning_rate": 6.774395102767203e-07,
- "loss": -0.03472680225968361,
- "num_tokens": 504906.0,
- "reward": 0.2722649872303009,
- "reward_std": 0.2922348082065582,
- "rewards/true_env_reward_fn/mean": 0.2722649872303009,
- "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
"step": 123,
- "step_time": 3.0233660449994204
+ "step_time": 8.471463828000196
},
{
"clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2819936871528625,
- "epoch": 1.008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00023045104171615094,
- "kl": 2.2608143808611203e-05,
- "learning_rate": 6.747005244854004e-07,
- "loss": 1.1284330412308918e-06,
- "num_tokens": 508329.0,
- "reward": 0.3149532079696655,
- "reward_std": 0.4275679290294647,
- "rewards/true_env_reward_fn/mean": 0.3149532079696655,
- "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
"step": 124,
- "step_time": 4.01701365199915
+ "step_time": 13.438758649999727
},
{
"clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 157.0,
- "completions/max_terminated_length": 157.0,
- "completions/mean_length": 80.875,
- "completions/mean_terminated_length": 80.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1542360186576843,
- "epoch": 1.016260162601626,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.232049003709108e-05,
- "kl": 1.291002809011843e-05,
- "learning_rate": 6.719369418252023e-07,
- "loss": 6.488799613180163e-07,
- "num_tokens": 515076.0,
- "reward": 0.4841846525669098,
- "reward_std": 0.12780573964118958,
- "rewards/true_env_reward_fn/mean": 0.4841846525669098,
- "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
"step": 125,
- "step_time": 7.240956699999515
+ "step_time": 8.654177170999901
},
{
"clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2594389915466309,
- "epoch": 1.024390243902439,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13091468811035156,
- "kl": 1.751603304001037e-05,
- "learning_rate": 6.691490097491675e-07,
- "loss": -0.033413223922252655,
- "num_tokens": 520279.0,
- "reward": 0.28095200657844543,
- "reward_std": 0.21837711334228516,
- "rewards/true_env_reward_fn/mean": 0.28095200657844543,
- "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
"step": 126,
- "step_time": 3.355879656997786
+ "step_time": 10.513378469000145
},
{
"clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 136.0,
- "completions/max_terminated_length": 136.0,
- "completions/mean_length": 77.875,
- "completions/mean_terminated_length": 77.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.3044686317443848,
- "epoch": 1.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12953205406665802,
- "kl": 1.8700401597016025e-05,
- "learning_rate": 6.663369778906008e-07,
- "loss": 0.03562816232442856,
- "num_tokens": 524582.0,
- "reward": 0.4330660402774811,
- "reward_std": 0.4592672288417816,
- "rewards/true_env_reward_fn/mean": 0.4330660402774811,
- "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
"step": 127,
- "step_time": 5.965807722999671
+ "step_time": 23.20779430600078
},
{
"clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 117.0,
- "completions/max_terminated_length": 117.0,
- "completions/mean_length": 75.375,
- "completions/mean_terminated_length": 75.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1742327809333801,
- "epoch": 1.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14993594586849213,
- "kl": 1.1459212601039326e-05,
- "learning_rate": 6.635010980407174e-07,
- "loss": 0.03646668791770935,
- "num_tokens": 526213.0,
- "reward": 0.7185037136077881,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.7185037136077881,
- "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
"step": 128,
- "step_time": 4.9305356690001645
+ "step_time": 11.551069149999876
},
{
"clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 49.125,
- "completions/mean_terminated_length": 49.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0784690976142883,
- "epoch": 1.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16007214784622192,
- "kl": 1.2491957932070363e-05,
- "learning_rate": 6.606416241260979e-07,
- "loss": 0.006608985364437103,
- "num_tokens": 531862.0,
- "reward": 0.2934249937534332,
- "reward_std": 0.2395382523536682,
- "rewards/true_env_reward_fn/mean": 0.2934249937534332,
- "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
"step": 129,
- "step_time": 3.173622508000335
+ "step_time": 10.723005456000465
},
{
"clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.308219850063324,
- "epoch": 1.056910569105691,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.985446427483112e-05,
- "kl": 1.2420873190421844e-05,
- "learning_rate": 6.577588121859508e-07,
- "loss": 6.241918413252279e-07,
- "num_tokens": 535957.0,
- "reward": 0.4817493259906769,
- "reward_std": 0.029202036559581757,
- "rewards/true_env_reward_fn/mean": 0.4817493259906769,
- "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
"step": 130,
- "step_time": 4.251137947001553
+ "step_time": 15.15810051499966
},
{
"clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.0767641067504883,
- "epoch": 1.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15786457061767578,
- "kl": 1.8847958926926367e-05,
- "learning_rate": 6.548529203491875e-07,
- "loss": -0.0026272237300872803,
- "num_tokens": 539269.0,
- "reward": 0.536803662776947,
- "reward_std": 0.30375123023986816,
- "rewards/true_env_reward_fn/mean": 0.536803662776947,
- "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
"step": 131,
- "step_time": 3.7980547870010923
+ "step_time": 20.109428818001106
},
{
"clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3295028805732727,
- "epoch": 1.0731707317073171,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1881481558084488,
- "kl": 2.0969039724150207e-05,
- "learning_rate": 6.519242088113085e-07,
- "loss": 0.08431969583034515,
- "num_tokens": 545691.0,
- "reward": 0.24590599536895752,
- "reward_std": 0.2047487199306488,
- "rewards/true_env_reward_fn/mean": 0.24590599536895752,
- "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
"step": 132,
- "step_time": 4.361092664001262
+ "step_time": 10.12403799699996
},
{
"clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2131375670433044,
- "epoch": 1.08130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13930389285087585,
- "kl": 1.1046585314034019e-05,
- "learning_rate": 6.489729398111058e-07,
- "loss": -0.03801802545785904,
- "num_tokens": 550295.0,
- "reward": 0.3215479254722595,
- "reward_std": 0.1736886352300644,
- "rewards/true_env_reward_fn/mean": 0.3215479254722595,
- "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
"step": 133,
- "step_time": 3.372364626999115
+ "step_time": 10.396350653999434
},
{
"clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 63.375,
- "completions/mean_terminated_length": 63.375,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2786019444465637,
- "epoch": 1.089430894308943,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12932609021663666,
- "kl": 1.340499647994875e-05,
- "learning_rate": 6.459993776071815e-07,
- "loss": 0.029022663831710815,
- "num_tokens": 553826.0,
- "reward": 0.4830188751220703,
- "reward_std": 0.29014864563941956,
- "rewards/true_env_reward_fn/mean": 0.4830188751220703,
- "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
"step": 134,
- "step_time": 3.215292060998763
+ "step_time": 10.961974539000039
},
{
"clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 0.9003906548023224,
- "epoch": 1.0975609756097562,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15958240628242493,
- "kl": 2.838099044311093e-05,
- "learning_rate": 6.430037884542861e-07,
- "loss": 0.11459673941135406,
- "num_tokens": 557217.0,
- "reward": 0.494448184967041,
- "reward_std": 0.3076546788215637,
- "rewards/true_env_reward_fn/mean": 0.494448184967041,
- "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
"step": 135,
- "step_time": 3.500462582000182
+ "step_time": 11.779171687000144
},
{
"clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 93.0,
"completions/max_terminated_length": 93.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2928712964057922,
- "epoch": 1.1056910569105691,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23077522218227386,
- "kl": 2.251418845844455e-05,
- "learning_rate": 6.399864405794782e-07,
- "loss": -0.05874824523925781,
- "num_tokens": 562421.0,
- "reward": 0.2385583370923996,
- "reward_std": 0.23380905389785767,
- "rewards/true_env_reward_fn/mean": 0.2385583370923996,
- "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
"step": 136,
- "step_time": 4.208805245998519
+ "step_time": 8.779588141000204
},
{
"clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8736326098442078,
- "epoch": 1.113821138211382,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.871674071997404e-05,
- "kl": 1.1485328741400735e-05,
- "learning_rate": 6.369476041581066e-07,
- "loss": 5.747077125306532e-07,
- "num_tokens": 566387.0,
- "reward": 0.4902166724205017,
- "reward_std": 0.038254011422395706,
- "rewards/true_env_reward_fn/mean": 0.4902166724205017,
- "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
"step": 137,
- "step_time": 3.981489739000608
+ "step_time": 11.74765996799988
},
{
"clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1538971662521362,
- "epoch": 1.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14676779508590698,
- "kl": 1.1651037766569061e-05,
- "learning_rate": 6.338875512896188e-07,
- "loss": 0.1347643882036209,
- "num_tokens": 569341.0,
- "reward": 0.43844783306121826,
- "reward_std": 0.16067014634609222,
- "rewards/true_env_reward_fn/mean": 0.43844783306121826,
- "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
"step": 138,
- "step_time": 3.689221037999232
+ "step_time": 8.8135579650002
},
{
"clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1961707472801208,
- "epoch": 1.1300813008130082,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16650564968585968,
- "kl": 1.4349476259667426e-05,
- "learning_rate": 6.308065559731976e-07,
- "loss": 0.007910434156656265,
- "num_tokens": 574046.0,
- "reward": 0.4596000015735626,
- "reward_std": 0.07715634256601334,
- "rewards/true_env_reward_fn/mean": 0.4596000015735626,
- "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
"step": 139,
- "step_time": 3.6711935700004688
+ "step_time": 9.997661417000472
},
{
"clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2013322114944458,
- "epoch": 1.1382113821138211,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14415834844112396,
- "kl": 1.4664098671346437e-05,
- "learning_rate": 6.277048940832264e-07,
- "loss": -0.016162052750587463,
- "num_tokens": 576769.0,
- "reward": 0.6152583360671997,
- "reward_std": 0.07727260142564774,
- "rewards/true_env_reward_fn/mean": 0.6152583360671997,
- "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
"step": 140,
- "step_time": 3.5191362610003125
+ "step_time": 14.679971276999822
},
{
"clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.75,
- "completions/mean_terminated_length": 44.75,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0287770330905914,
- "epoch": 1.146341463414634,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.892051457427442e-05,
- "kl": 1.1797974821092794e-05,
- "learning_rate": 6.245828433445872e-07,
- "loss": 5.92092192164273e-07,
- "num_tokens": 578843.0,
- "reward": 0.6387845277786255,
- "reward_std": 0.13867565989494324,
- "rewards/true_env_reward_fn/mean": 0.6387845277786255,
- "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
"step": 141,
- "step_time": 2.5015027329991426
+ "step_time": 11.64747691499997
},
{
"clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1590029001235962,
- "epoch": 1.1544715447154472,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.160966694355011,
- "kl": 1.4735675904375967e-05,
- "learning_rate": 6.214406833077937e-07,
- "loss": 0.0170527845621109,
- "num_tokens": 583201.0,
- "reward": 0.36017733812332153,
- "reward_std": 0.3556094467639923,
- "rewards/true_env_reward_fn/mean": 0.36017733812332153,
- "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
"step": 142,
- "step_time": 3.2783409929998015
+ "step_time": 8.380270293999729
},
{
"clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 59.0,
- "completions/min_terminated_length": 59.0,
- "entropy": 1.1985241174697876,
- "epoch": 1.1626016260162602,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12272457778453827,
- "kl": 1.7849098185251933e-05,
- "learning_rate": 6.182786953239593e-07,
- "loss": -0.0016125142574310303,
- "num_tokens": 587317.0,
- "reward": 0.34745320677757263,
- "reward_std": 0.3954337239265442,
- "rewards/true_env_reward_fn/mean": 0.34745320677757263,
- "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
"step": 143,
- "step_time": 3.9932043310000154
+ "step_time": 11.716556537000088
},
{
"clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 48.0,
- "completions/max_terminated_length": 48.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1116944551467896,
- "epoch": 1.170731707317073,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18435250222682953,
- "kl": 1.014559029499651e-05,
- "learning_rate": 6.150971625196048e-07,
- "loss": 0.009793907403945923,
- "num_tokens": 590191.0,
- "reward": 0.4938516616821289,
- "reward_std": 0.03703190013766289,
- "rewards/true_env_reward_fn/mean": 0.4938516616821289,
- "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
"step": 144,
- "step_time": 2.3663663690022076
+ "step_time": 10.67718802499985
},
{
"clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.0,
- "completions/mean_terminated_length": 53.0,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.413200855255127,
- "epoch": 1.1788617886178863,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14336225390434265,
- "kl": 2.0541991034406237e-05,
- "learning_rate": 6.118963697713078e-07,
- "loss": -0.013927727937698364,
- "num_tokens": 593671.0,
- "reward": 0.4619143605232239,
- "reward_std": 0.3773181140422821,
- "rewards/true_env_reward_fn/mean": 0.4619143605232239,
- "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
"step": 145,
- "step_time": 3.9730388410007436
+ "step_time": 8.610043666999445
},
{
"clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 128.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 6.0,
- "completions/min_terminated_length": 6.0,
- "entropy": 1.08676016330719,
- "epoch": 1.1869918699186992,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15555191040039062,
- "kl": 1.6947700260061538e-05,
- "learning_rate": 6.086766036801937e-07,
- "loss": -0.139797180891037,
- "num_tokens": 601612.0,
- "reward": 0.3831036686897278,
- "reward_std": 0.09242849797010422,
- "rewards/true_env_reward_fn/mean": 0.3831036686897278,
- "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
"step": 146,
- "step_time": 6.323679949000507
+ "step_time": 13.649344002000362
},
{
"clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.5055813789367676,
- "epoch": 1.1951219512195121,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2397669553756714,
- "kl": 2.704876442294335e-05,
- "learning_rate": 6.054381525462745e-07,
- "loss": 0.2738838493824005,
- "num_tokens": 606712.0,
- "reward": 0.25339600443840027,
- "reward_std": 0.3023079037666321,
- "rewards/true_env_reward_fn/mean": 0.25339600443840027,
- "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
"step": 147,
- "step_time": 5.185072233998653
+ "step_time": 11.702765863000877
},
{
"clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 47.25,
- "completions/mean_terminated_length": 47.25,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.135968267917633,
- "epoch": 1.203252032520325,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2614514231681824,
- "kl": 3.613240005506668e-05,
- "learning_rate": 6.021813063426323e-07,
- "loss": 0.10286401212215424,
- "num_tokens": 610566.0,
- "reward": 0.31031692028045654,
- "reward_std": 0.3124054968357086,
- "rewards/true_env_reward_fn/mean": 0.31031692028045654,
- "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
"step": 148,
- "step_time": 3.2177847610000754
+ "step_time": 13.306644664000487
},
{
"clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 52.75,
- "completions/mean_terminated_length": 52.75,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4589928984642029,
- "epoch": 1.2113821138211383,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2019941508769989,
- "kl": 2.1841721718374174e-05,
- "learning_rate": 5.989063566894572e-07,
- "loss": 0.010915875434875488,
- "num_tokens": 615716.0,
- "reward": 0.31711751222610474,
- "reward_std": 0.13289952278137207,
- "rewards/true_env_reward_fn/mean": 0.31711751222610474,
- "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
"step": 149,
- "step_time": 4.3804878079990885
+ "step_time": 9.446422488999815
},
{
"clip_ratio/high_max": 0.0,
@@ -4188,32 +4188,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.1892729997634888,
- "epoch": 1.2195121951219512,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.835455471649766e-05,
- "kl": 1.3420096820482286e-05,
- "learning_rate": 5.956135968279332e-07,
- "loss": 6.646802717114042e-07,
- "num_tokens": 619439.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
"step": 150,
- "step_time": 3.63938895299907
+ "step_time": 10.538116119999813
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 619439,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 3702516,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -4228,7 +4228,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-150/training_args.bin b/checkpoint-150/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-150/training_args.bin
+++ b/checkpoint-150/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-200/adapter_model.safetensors b/checkpoint-200/adapter_model.safetensors
index 08a2c3e7ec2105e12a84d9cd57207946ec7a7573..435d6c77e602e167038eec4d709e1e84244ad730 100644
--- a/checkpoint-200/adapter_model.safetensors
+++ b/checkpoint-200/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:3760d3314549081af05923853132db52f91e157452d1a671fa01138e8ce0d7f6
+oid sha256:162d752136458d0b1563dc000c1303146bf581c1dc30d4ac1c6cf143ca55920c
size 8731128
diff --git a/checkpoint-200/optimizer.pt b/checkpoint-200/optimizer.pt
index c3a0d4e01f9555d126290e96a3d6470b7557b4c7..d9ed815faf7df87f8364e4ac59b108e75bcf495a 100644
--- a/checkpoint-200/optimizer.pt
+++ b/checkpoint-200/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:2a433c0070b886ab356b3b0e01ac77896089a29f3e1654b8040ce587b35da19b
+oid sha256:046dda077131549f583f0678216d0b9ac6eb6b0661463c77250bcb0557eeec5c
size 17526842
diff --git a/checkpoint-200/ref/adapter_model.safetensors b/checkpoint-200/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-200/ref/adapter_model.safetensors
+++ b/checkpoint-200/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-200/rng_state.pth b/checkpoint-200/rng_state.pth
index 23ee49c41a13b857dd690cbd634f7d0f10e60508..141e053517cd3cf78317801b6d531190ff51fcb8 100644
--- a/checkpoint-200/rng_state.pth
+++ b/checkpoint-200/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:f4825bf31044ed1c1141fad5e2d04ce4a1b0a47c2a63289a50920eb60f313e05
+oid sha256:57846bf5428a3c03cedb89f5e74e3f1f6343bc7406d84cce8f4144fb81d70df6
size 14244
diff --git a/checkpoint-200/scheduler.pt b/checkpoint-200/scheduler.pt
index a081c242fdc3d41cb596d73cee2ca70aa1e1efe8..9b5a119d4d9c9213c177752dd64ff7a3395e3553 100644
--- a/checkpoint-200/scheduler.pt
+++ b/checkpoint-200/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:820dc20ea8497c148d31704bc1cfea7c0d598f4fe3e52b035416778336d712a4
+oid sha256:8a67c94c0e1a9875787830b5a63a3dc9c7b612c1523eacffed9a7583a89f7bef
size 1064
diff --git a/checkpoint-200/trainer_state.json b/checkpoint-200/trainer_state.json
index 2e50dab06b25d3c046c838d74e61d2f515cd2401..27a942f50f795d19b8d2020713c0c89d1a4be8f3 100644
--- a/checkpoint-200/trainer_state.json
+++ b/checkpoint-200/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 1.6260162601626016,
+ "epoch": 4.878048780487805,
"eval_steps": 500,
"global_step": 200,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,54 +2144,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
- "clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
},
{
"clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.0,
- "completions/mean_terminated_length": 59.0,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2509461045265198,
- "epoch": 0.8211382113821138,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22887632250785828,
- "kl": 2.1612477212329395e-05,
- "learning_rate": 7.310023360571047e-07,
- "loss": 0.025605827569961548,
- "num_tokens": 414080.0,
- "reward": 0.588032603263855,
- "reward_std": 0.11032751202583313,
- "rewards/true_env_reward_fn/mean": 0.588032603263855,
- "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
"step": 101,
- "step_time": 3.625197022998691
+ "step_time": 16.478299477000064
},
{
"clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 176.0,
- "completions/max_terminated_length": 176.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5443179607391357,
- "epoch": 0.8292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10614532232284546,
- "kl": 1.492139062975184e-05,
- "learning_rate": 7.28862404677924e-07,
- "loss": 0.06531564146280289,
- "num_tokens": 419835.0,
- "reward": 0.07074306160211563,
- "reward_std": 0.2918013632297516,
- "rewards/true_env_reward_fn/mean": 0.07074306160211563,
- "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
"step": 102,
- "step_time": 7.796810614998321
+ "step_time": 11.344593008000174
},
{
"clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 101.0,
"completions/max_terminated_length": 101.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3223788738250732,
- "epoch": 0.8373983739837398,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2421368807554245,
- "kl": 3.265505938543356e-05,
- "learning_rate": 7.266930267388503e-07,
- "loss": -0.07752113044261932,
- "num_tokens": 422773.0,
- "reward": 0.33568501472473145,
- "reward_std": 0.2780380845069885,
- "rewards/true_env_reward_fn/mean": 0.33568501472473145,
- "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
"step": 103,
- "step_time": 4.313938073000827
+ "step_time": 8.135681302999728
},
{
"clip_ratio/high_max": 0.0,
@@ -2900,26 +2900,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 67.25,
- "completions/mean_terminated_length": 67.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.3332037329673767,
- "epoch": 0.8455284552845529,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13883370161056519,
- "kl": 2.1224042484391248e-05,
- "learning_rate": 7.244943964874369e-07,
- "loss": 0.021739646792411804,
- "num_tokens": 426507.0,
- "reward": 0.40595096349716187,
- "reward_std": 0.2035457342863083,
- "rewards/true_env_reward_fn/mean": 0.40595096349716187,
- "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
"step": 104,
- "step_time": 4.155937195999286
+ "step_time": 15.185034105999875
},
{
"clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
"completions/min_length": 31.0,
"completions/min_terminated_length": 31.0,
- "entropy": 1.3391229510307312,
- "epoch": 0.8536585365853658,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24205009639263153,
- "kl": 2.5022183763212524e-05,
- "learning_rate": 7.222667107905085e-07,
- "loss": 0.06330433487892151,
- "num_tokens": 429010.0,
- "reward": 0.3355163037776947,
- "reward_std": 0.2902730703353882,
- "rewards/true_env_reward_fn/mean": 0.3355163037776947,
- "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
"step": 105,
- "step_time": 3.808478789000219
+ "step_time": 9.588520330999927
},
{
"clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1864720582962036,
- "epoch": 0.8617886178861789,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13473568856716156,
- "kl": 1.4212585938366828e-05,
- "learning_rate": 7.200101691165338e-07,
- "loss": -0.020715661346912384,
- "num_tokens": 432403.0,
- "reward": 0.4871198534965515,
- "reward_std": 0.15407639741897583,
- "rewards/true_env_reward_fn/mean": 0.4871198534965515,
- "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
"step": 106,
- "step_time": 4.240638332001254
+ "step_time": 14.60399662399982
},
{
"clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.0,
- "completions/mean_terminated_length": 48.0,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "entropy": 1.0669284462928772,
- "epoch": 0.8699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14131899178028107,
- "kl": 1.5787159554747632e-05,
- "learning_rate": 7.177249735177651e-07,
- "loss": 0.03678784519433975,
- "num_tokens": 435995.0,
- "reward": 0.5010770559310913,
- "reward_std": 0.48966261744499207,
- "rewards/true_env_reward_fn/mean": 0.5010770559310913,
- "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
"step": 107,
- "step_time": 3.3587191269998584
+ "step_time": 20.100954443999854
},
{
"clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 71.875,
- "completions/mean_terminated_length": 71.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.304731547832489,
- "epoch": 0.8780487804878049,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.380985673284158e-05,
- "kl": 1.3128728824085556e-05,
- "learning_rate": 7.154113286121462e-07,
- "loss": 6.494262834166875e-07,
- "num_tokens": 442094.0,
- "reward": 0.4055500030517578,
- "reward_std": 0.052258480340242386,
- "rewards/true_env_reward_fn/mean": 0.4055500030517578,
- "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
"step": 108,
- "step_time": 4.337008413998774
+ "step_time": 8.304149297999857
},
{
"clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2324069738388062,
- "epoch": 0.8861788617886179,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13588950037956238,
- "kl": 1.3448377558233915e-05,
- "learning_rate": 7.130694415649912e-07,
- "loss": 1.0952353477478027e-06,
- "num_tokens": 447226.0,
- "reward": 0.20854972302913666,
- "reward_std": 0.06059705466032028,
- "rewards/true_env_reward_fn/mean": 0.20854972302913666,
- "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
"step": 109,
- "step_time": 3.2976038649994734
+ "step_time": 11.260021517000041
},
{
"clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1567262411117554,
- "epoch": 0.8943089430894309,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.736967720324174e-05,
- "kl": 1.2838129805459175e-05,
- "learning_rate": 7.106995220704342e-07,
- "loss": 6.425898391171359e-07,
- "num_tokens": 450359.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
"step": 110,
- "step_time": 4.067084037998939
+ "step_time": 22.80178854000019
},
{
"clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 100.0,
"completions/max_terminated_length": 100.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 12.0,
- "completions/min_terminated_length": 12.0,
- "entropy": 1.496058464050293,
- "epoch": 0.9024390243902439,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18041981756687164,
- "kl": 1.6616825632809196e-05,
- "learning_rate": 7.083017823326532e-07,
- "loss": 0.0269068144261837,
- "num_tokens": 453583.0,
- "reward": 0.5647265911102295,
- "reward_std": 0.1507105529308319,
- "rewards/true_env_reward_fn/mean": 0.5647265911102295,
- "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
"step": 111,
- "step_time": 4.347732382997492
+ "step_time": 9.663163859000178
},
{
"clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 67.75,
- "completions/mean_terminated_length": 67.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.4196155667304993,
- "epoch": 0.9105691056910569,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18451112508773804,
- "kl": 2.1803500203532167e-05,
- "learning_rate": 7.058764370468698e-07,
- "loss": 0.1650262475013733,
- "num_tokens": 456773.0,
- "reward": 0.6907394528388977,
- "reward_std": 0.1393815129995346,
- "rewards/true_env_reward_fn/mean": 0.6907394528388977,
- "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
"step": 112,
- "step_time": 4.627644968999448
+ "step_time": 9.903081222999617
},
{
"clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2243221998214722,
- "epoch": 0.9186991869918699,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331371545791626,
- "kl": 1.548633599668392e-05,
- "learning_rate": 7.034237033801247e-07,
- "loss": 0.039844345301389694,
- "num_tokens": 462714.0,
- "reward": 0.21676866710186005,
- "reward_std": 0.26559779047966003,
- "rewards/true_env_reward_fn/mean": 0.21676866710186005,
- "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
"step": 113,
- "step_time": 3.8455466220002563
+ "step_time": 10.101770388999284
},
{
"clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 73.0,
- "completions/mean_terminated_length": 73.0,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.325823724269867,
- "epoch": 0.926829268292683,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1866220384836197,
- "kl": 1.8801019905367866e-05,
- "learning_rate": 7.009438009518325e-07,
- "loss": 0.06504581868648529,
- "num_tokens": 465994.0,
- "reward": 0.5194582939147949,
- "reward_std": 0.2796703577041626,
- "rewards/true_env_reward_fn/mean": 0.5194582939147949,
- "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
"step": 114,
- "step_time": 4.151028698999653
+ "step_time": 17.633509853000305
},
{
"clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1521879434585571,
- "epoch": 0.9349593495934959,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14320029318332672,
- "kl": 1.2749982033710694e-05,
- "learning_rate": 6.98436951814117e-07,
- "loss": 0.03685300797224045,
- "num_tokens": 468615.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
"step": 115,
- "step_time": 3.6973990600017714
+ "step_time": 12.307247350000125
},
{
"clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.201507806777954,
- "epoch": 0.943089430894309,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14264807105064392,
- "kl": 2.6679515940486453e-05,
- "learning_rate": 6.959033804319283e-07,
- "loss": -0.023484818637371063,
- "num_tokens": 471647.0,
- "reward": 0.41836902499198914,
- "reward_std": 0.3116860091686249,
- "rewards/true_env_reward_fn/mean": 0.41836902499198914,
- "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
"step": 116,
- "step_time": 3.1295652919998247
+ "step_time": 14.468690254000194
},
{
"clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 62.375,
- "completions/mean_terminated_length": 62.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2834057807922363,
- "epoch": 0.9512195121951219,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.810227154754102e-05,
- "kl": 1.4841665233689127e-05,
- "learning_rate": 6.933433136629443e-07,
- "loss": 7.425555850204546e-07,
- "num_tokens": 474682.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
"step": 117,
- "step_time": 3.4368692790012574
+ "step_time": 11.849063975999798
},
{
"clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.191932499408722,
- "epoch": 0.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21204856038093567,
- "kl": 3.64198385796044e-05,
- "learning_rate": 6.907569807372574e-07,
- "loss": -0.001312553882598877,
- "num_tokens": 477027.0,
- "reward": 0.5300568342208862,
- "reward_std": 0.2945883274078369,
- "rewards/true_env_reward_fn/mean": 0.5300568342208862,
- "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
"step": 118,
- "step_time": 3.8569856240010267
+ "step_time": 16.597334930000216
},
{
"clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 190.0,
- "completions/max_terminated_length": 190.0,
- "completions/mean_length": 96.5,
- "completions/mean_terminated_length": 96.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2401175498962402,
- "epoch": 0.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011377666669432074,
- "kl": 1.3742283954343293e-05,
- "learning_rate": 6.881446132368494e-07,
- "loss": 6.866695230201003e-07,
- "num_tokens": 481999.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
"step": 119,
- "step_time": 8.09440958399864
+ "step_time": 11.729475523999554
},
{
"clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.230682611465454,
- "epoch": 0.975609756097561,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22175048291683197,
- "kl": 1.2522132237791084e-05,
- "learning_rate": 6.855064450748555e-07,
- "loss": -0.04083740711212158,
- "num_tokens": 490884.0,
- "reward": 0.13476666808128357,
- "reward_std": 0.2987530529499054,
- "rewards/true_env_reward_fn/mean": 0.13476666808128357,
- "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
"step": 120,
- "step_time": 4.678523641001448
+ "step_time": 11.503627788000358
},
{
"clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2799639105796814,
- "epoch": 0.983739837398374,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19926966726779938,
- "kl": 1.7022688552970067e-05,
- "learning_rate": 6.828427124746189e-07,
- "loss": -0.010804429650306702,
- "num_tokens": 496404.0,
- "reward": 0.24633333086967468,
- "reward_std": 0.2454334795475006,
- "rewards/true_env_reward_fn/mean": 0.24633333086967468,
- "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
"step": 121,
- "step_time": 3.98071062300005
+ "step_time": 8.728293746000418
},
{
"clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 46.875,
- "completions/mean_terminated_length": 46.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3840235471725464,
- "epoch": 0.991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24853822588920593,
- "kl": 3.688259130285587e-05,
- "learning_rate": 6.801536539485403e-07,
- "loss": 0.10205884277820587,
- "num_tokens": 499767.0,
- "reward": 0.3045905530452728,
- "reward_std": 0.262839138507843,
- "rewards/true_env_reward_fn/mean": 0.3045905530452728,
- "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
"step": 122,
- "step_time": 3.3792565210005705
+ "step_time": 15.51957702100026
},
{
"clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 50.375,
- "completions/mean_terminated_length": 50.375,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2064164280891418,
- "epoch": 1.0,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1365528553724289,
- "kl": 2.434901080050622e-05,
- "learning_rate": 6.774395102767203e-07,
- "loss": -0.03472680225968361,
- "num_tokens": 504906.0,
- "reward": 0.2722649872303009,
- "reward_std": 0.2922348082065582,
- "rewards/true_env_reward_fn/mean": 0.2722649872303009,
- "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
"step": 123,
- "step_time": 3.0233660449994204
+ "step_time": 8.471463828000196
},
{
"clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2819936871528625,
- "epoch": 1.008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00023045104171615094,
- "kl": 2.2608143808611203e-05,
- "learning_rate": 6.747005244854004e-07,
- "loss": 1.1284330412308918e-06,
- "num_tokens": 508329.0,
- "reward": 0.3149532079696655,
- "reward_std": 0.4275679290294647,
- "rewards/true_env_reward_fn/mean": 0.3149532079696655,
- "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
"step": 124,
- "step_time": 4.01701365199915
+ "step_time": 13.438758649999727
},
{
"clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 157.0,
- "completions/max_terminated_length": 157.0,
- "completions/mean_length": 80.875,
- "completions/mean_terminated_length": 80.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1542360186576843,
- "epoch": 1.016260162601626,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.232049003709108e-05,
- "kl": 1.291002809011843e-05,
- "learning_rate": 6.719369418252023e-07,
- "loss": 6.488799613180163e-07,
- "num_tokens": 515076.0,
- "reward": 0.4841846525669098,
- "reward_std": 0.12780573964118958,
- "rewards/true_env_reward_fn/mean": 0.4841846525669098,
- "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
"step": 125,
- "step_time": 7.240956699999515
+ "step_time": 8.654177170999901
},
{
"clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2594389915466309,
- "epoch": 1.024390243902439,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13091468811035156,
- "kl": 1.751603304001037e-05,
- "learning_rate": 6.691490097491675e-07,
- "loss": -0.033413223922252655,
- "num_tokens": 520279.0,
- "reward": 0.28095200657844543,
- "reward_std": 0.21837711334228516,
- "rewards/true_env_reward_fn/mean": 0.28095200657844543,
- "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
"step": 126,
- "step_time": 3.355879656997786
+ "step_time": 10.513378469000145
},
{
"clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 136.0,
- "completions/max_terminated_length": 136.0,
- "completions/mean_length": 77.875,
- "completions/mean_terminated_length": 77.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.3044686317443848,
- "epoch": 1.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12953205406665802,
- "kl": 1.8700401597016025e-05,
- "learning_rate": 6.663369778906008e-07,
- "loss": 0.03562816232442856,
- "num_tokens": 524582.0,
- "reward": 0.4330660402774811,
- "reward_std": 0.4592672288417816,
- "rewards/true_env_reward_fn/mean": 0.4330660402774811,
- "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
"step": 127,
- "step_time": 5.965807722999671
+ "step_time": 23.20779430600078
},
{
"clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 117.0,
- "completions/max_terminated_length": 117.0,
- "completions/mean_length": 75.375,
- "completions/mean_terminated_length": 75.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1742327809333801,
- "epoch": 1.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14993594586849213,
- "kl": 1.1459212601039326e-05,
- "learning_rate": 6.635010980407174e-07,
- "loss": 0.03646668791770935,
- "num_tokens": 526213.0,
- "reward": 0.7185037136077881,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.7185037136077881,
- "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
"step": 128,
- "step_time": 4.9305356690001645
+ "step_time": 11.551069149999876
},
{
"clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 49.125,
- "completions/mean_terminated_length": 49.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0784690976142883,
- "epoch": 1.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16007214784622192,
- "kl": 1.2491957932070363e-05,
- "learning_rate": 6.606416241260979e-07,
- "loss": 0.006608985364437103,
- "num_tokens": 531862.0,
- "reward": 0.2934249937534332,
- "reward_std": 0.2395382523536682,
- "rewards/true_env_reward_fn/mean": 0.2934249937534332,
- "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
"step": 129,
- "step_time": 3.173622508000335
+ "step_time": 10.723005456000465
},
{
"clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.308219850063324,
- "epoch": 1.056910569105691,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.985446427483112e-05,
- "kl": 1.2420873190421844e-05,
- "learning_rate": 6.577588121859508e-07,
- "loss": 6.241918413252279e-07,
- "num_tokens": 535957.0,
- "reward": 0.4817493259906769,
- "reward_std": 0.029202036559581757,
- "rewards/true_env_reward_fn/mean": 0.4817493259906769,
- "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
"step": 130,
- "step_time": 4.251137947001553
+ "step_time": 15.15810051499966
},
{
"clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.0767641067504883,
- "epoch": 1.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15786457061767578,
- "kl": 1.8847958926926367e-05,
- "learning_rate": 6.548529203491875e-07,
- "loss": -0.0026272237300872803,
- "num_tokens": 539269.0,
- "reward": 0.536803662776947,
- "reward_std": 0.30375123023986816,
- "rewards/true_env_reward_fn/mean": 0.536803662776947,
- "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
"step": 131,
- "step_time": 3.7980547870010923
+ "step_time": 20.109428818001106
},
{
"clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3295028805732727,
- "epoch": 1.0731707317073171,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1881481558084488,
- "kl": 2.0969039724150207e-05,
- "learning_rate": 6.519242088113085e-07,
- "loss": 0.08431969583034515,
- "num_tokens": 545691.0,
- "reward": 0.24590599536895752,
- "reward_std": 0.2047487199306488,
- "rewards/true_env_reward_fn/mean": 0.24590599536895752,
- "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
"step": 132,
- "step_time": 4.361092664001262
+ "step_time": 10.12403799699996
},
{
"clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2131375670433044,
- "epoch": 1.08130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13930389285087585,
- "kl": 1.1046585314034019e-05,
- "learning_rate": 6.489729398111058e-07,
- "loss": -0.03801802545785904,
- "num_tokens": 550295.0,
- "reward": 0.3215479254722595,
- "reward_std": 0.1736886352300644,
- "rewards/true_env_reward_fn/mean": 0.3215479254722595,
- "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
"step": 133,
- "step_time": 3.372364626999115
+ "step_time": 10.396350653999434
},
{
"clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 63.375,
- "completions/mean_terminated_length": 63.375,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2786019444465637,
- "epoch": 1.089430894308943,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12932609021663666,
- "kl": 1.340499647994875e-05,
- "learning_rate": 6.459993776071815e-07,
- "loss": 0.029022663831710815,
- "num_tokens": 553826.0,
- "reward": 0.4830188751220703,
- "reward_std": 0.29014864563941956,
- "rewards/true_env_reward_fn/mean": 0.4830188751220703,
- "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
"step": 134,
- "step_time": 3.215292060998763
+ "step_time": 10.961974539000039
},
{
"clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 0.9003906548023224,
- "epoch": 1.0975609756097562,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15958240628242493,
- "kl": 2.838099044311093e-05,
- "learning_rate": 6.430037884542861e-07,
- "loss": 0.11459673941135406,
- "num_tokens": 557217.0,
- "reward": 0.494448184967041,
- "reward_std": 0.3076546788215637,
- "rewards/true_env_reward_fn/mean": 0.494448184967041,
- "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
"step": 135,
- "step_time": 3.500462582000182
+ "step_time": 11.779171687000144
},
{
"clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 93.0,
"completions/max_terminated_length": 93.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2928712964057922,
- "epoch": 1.1056910569105691,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23077522218227386,
- "kl": 2.251418845844455e-05,
- "learning_rate": 6.399864405794782e-07,
- "loss": -0.05874824523925781,
- "num_tokens": 562421.0,
- "reward": 0.2385583370923996,
- "reward_std": 0.23380905389785767,
- "rewards/true_env_reward_fn/mean": 0.2385583370923996,
- "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
"step": 136,
- "step_time": 4.208805245998519
+ "step_time": 8.779588141000204
},
{
"clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8736326098442078,
- "epoch": 1.113821138211382,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.871674071997404e-05,
- "kl": 1.1485328741400735e-05,
- "learning_rate": 6.369476041581066e-07,
- "loss": 5.747077125306532e-07,
- "num_tokens": 566387.0,
- "reward": 0.4902166724205017,
- "reward_std": 0.038254011422395706,
- "rewards/true_env_reward_fn/mean": 0.4902166724205017,
- "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
"step": 137,
- "step_time": 3.981489739000608
+ "step_time": 11.74765996799988
},
{
"clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1538971662521362,
- "epoch": 1.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14676779508590698,
- "kl": 1.1651037766569061e-05,
- "learning_rate": 6.338875512896188e-07,
- "loss": 0.1347643882036209,
- "num_tokens": 569341.0,
- "reward": 0.43844783306121826,
- "reward_std": 0.16067014634609222,
- "rewards/true_env_reward_fn/mean": 0.43844783306121826,
- "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
"step": 138,
- "step_time": 3.689221037999232
+ "step_time": 8.8135579650002
},
{
"clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1961707472801208,
- "epoch": 1.1300813008130082,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16650564968585968,
- "kl": 1.4349476259667426e-05,
- "learning_rate": 6.308065559731976e-07,
- "loss": 0.007910434156656265,
- "num_tokens": 574046.0,
- "reward": 0.4596000015735626,
- "reward_std": 0.07715634256601334,
- "rewards/true_env_reward_fn/mean": 0.4596000015735626,
- "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
"step": 139,
- "step_time": 3.6711935700004688
+ "step_time": 9.997661417000472
},
{
"clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2013322114944458,
- "epoch": 1.1382113821138211,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14415834844112396,
- "kl": 1.4664098671346437e-05,
- "learning_rate": 6.277048940832264e-07,
- "loss": -0.016162052750587463,
- "num_tokens": 576769.0,
- "reward": 0.6152583360671997,
- "reward_std": 0.07727260142564774,
- "rewards/true_env_reward_fn/mean": 0.6152583360671997,
- "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
"step": 140,
- "step_time": 3.5191362610003125
+ "step_time": 14.679971276999822
},
{
"clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.75,
- "completions/mean_terminated_length": 44.75,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0287770330905914,
- "epoch": 1.146341463414634,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.892051457427442e-05,
- "kl": 1.1797974821092794e-05,
- "learning_rate": 6.245828433445872e-07,
- "loss": 5.92092192164273e-07,
- "num_tokens": 578843.0,
- "reward": 0.6387845277786255,
- "reward_std": 0.13867565989494324,
- "rewards/true_env_reward_fn/mean": 0.6387845277786255,
- "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
"step": 141,
- "step_time": 2.5015027329991426
+ "step_time": 11.64747691499997
},
{
"clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1590029001235962,
- "epoch": 1.1544715447154472,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.160966694355011,
- "kl": 1.4735675904375967e-05,
- "learning_rate": 6.214406833077937e-07,
- "loss": 0.0170527845621109,
- "num_tokens": 583201.0,
- "reward": 0.36017733812332153,
- "reward_std": 0.3556094467639923,
- "rewards/true_env_reward_fn/mean": 0.36017733812332153,
- "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
"step": 142,
- "step_time": 3.2783409929998015
+ "step_time": 8.380270293999729
},
{
"clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 59.0,
- "completions/min_terminated_length": 59.0,
- "entropy": 1.1985241174697876,
- "epoch": 1.1626016260162602,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12272457778453827,
- "kl": 1.7849098185251933e-05,
- "learning_rate": 6.182786953239593e-07,
- "loss": -0.0016125142574310303,
- "num_tokens": 587317.0,
- "reward": 0.34745320677757263,
- "reward_std": 0.3954337239265442,
- "rewards/true_env_reward_fn/mean": 0.34745320677757263,
- "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
"step": 143,
- "step_time": 3.9932043310000154
+ "step_time": 11.716556537000088
},
{
"clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 48.0,
- "completions/max_terminated_length": 48.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1116944551467896,
- "epoch": 1.170731707317073,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18435250222682953,
- "kl": 1.014559029499651e-05,
- "learning_rate": 6.150971625196048e-07,
- "loss": 0.009793907403945923,
- "num_tokens": 590191.0,
- "reward": 0.4938516616821289,
- "reward_std": 0.03703190013766289,
- "rewards/true_env_reward_fn/mean": 0.4938516616821289,
- "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
"step": 144,
- "step_time": 2.3663663690022076
+ "step_time": 10.67718802499985
},
{
"clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.0,
- "completions/mean_terminated_length": 53.0,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.413200855255127,
- "epoch": 1.1788617886178863,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14336225390434265,
- "kl": 2.0541991034406237e-05,
- "learning_rate": 6.118963697713078e-07,
- "loss": -0.013927727937698364,
- "num_tokens": 593671.0,
- "reward": 0.4619143605232239,
- "reward_std": 0.3773181140422821,
- "rewards/true_env_reward_fn/mean": 0.4619143605232239,
- "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
"step": 145,
- "step_time": 3.9730388410007436
+ "step_time": 8.610043666999445
},
{
"clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 128.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 6.0,
- "completions/min_terminated_length": 6.0,
- "entropy": 1.08676016330719,
- "epoch": 1.1869918699186992,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15555191040039062,
- "kl": 1.6947700260061538e-05,
- "learning_rate": 6.086766036801937e-07,
- "loss": -0.139797180891037,
- "num_tokens": 601612.0,
- "reward": 0.3831036686897278,
- "reward_std": 0.09242849797010422,
- "rewards/true_env_reward_fn/mean": 0.3831036686897278,
- "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
"step": 146,
- "step_time": 6.323679949000507
+ "step_time": 13.649344002000362
},
{
"clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.5055813789367676,
- "epoch": 1.1951219512195121,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2397669553756714,
- "kl": 2.704876442294335e-05,
- "learning_rate": 6.054381525462745e-07,
- "loss": 0.2738838493824005,
- "num_tokens": 606712.0,
- "reward": 0.25339600443840027,
- "reward_std": 0.3023079037666321,
- "rewards/true_env_reward_fn/mean": 0.25339600443840027,
- "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
"step": 147,
- "step_time": 5.185072233998653
+ "step_time": 11.702765863000877
},
{
"clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 47.25,
- "completions/mean_terminated_length": 47.25,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.135968267917633,
- "epoch": 1.203252032520325,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2614514231681824,
- "kl": 3.613240005506668e-05,
- "learning_rate": 6.021813063426323e-07,
- "loss": 0.10286401212215424,
- "num_tokens": 610566.0,
- "reward": 0.31031692028045654,
- "reward_std": 0.3124054968357086,
- "rewards/true_env_reward_fn/mean": 0.31031692028045654,
- "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
"step": 148,
- "step_time": 3.2177847610000754
+ "step_time": 13.306644664000487
},
{
"clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 52.75,
- "completions/mean_terminated_length": 52.75,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4589928984642029,
- "epoch": 1.2113821138211383,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2019941508769989,
- "kl": 2.1841721718374174e-05,
- "learning_rate": 5.989063566894572e-07,
- "loss": 0.010915875434875488,
- "num_tokens": 615716.0,
- "reward": 0.31711751222610474,
- "reward_std": 0.13289952278137207,
- "rewards/true_env_reward_fn/mean": 0.31711751222610474,
- "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
"step": 149,
- "step_time": 4.3804878079990885
+ "step_time": 9.446422488999815
},
{
"clip_ratio/high_max": 0.0,
@@ -4188,26 +4188,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.1892729997634888,
- "epoch": 1.2195121951219512,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.835455471649766e-05,
- "kl": 1.3420096820482286e-05,
- "learning_rate": 5.956135968279332e-07,
- "loss": 6.646802717114042e-07,
- "num_tokens": 619439.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
"step": 150,
- "step_time": 3.63938895299907
+ "step_time": 10.538116119999813
},
{
"clip_ratio/high_max": 0.0,
@@ -4216,26 +4216,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1402934789657593,
- "epoch": 1.2276422764227641,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010795716661959887,
- "kl": 1.4652535810455447e-05,
- "learning_rate": 5.923033215939834e-07,
- "loss": 7.542968205598299e-07,
- "num_tokens": 621009.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
"step": 151,
- "step_time": 3.926544339999964
+ "step_time": 15.755764130999978
},
{
"clip_ratio/high_max": 0.0,
@@ -4244,26 +4244,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
"completions/mean_length": 64.125,
"completions/mean_terminated_length": 64.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.190350890159607,
- "epoch": 1.2357723577235773,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19853363931179047,
- "kl": 2.269768037876929e-05,
- "learning_rate": 5.889758273918683e-07,
- "loss": 0.044217392802238464,
- "num_tokens": 623994.0,
- "reward": 0.4411996603012085,
- "reward_std": 0.2517909109592438,
- "rewards/true_env_reward_fn/mean": 0.4411996603012085,
- "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
"step": 152,
- "step_time": 3.7339736520007136
+ "step_time": 9.670861957999932
},
{
"clip_ratio/high_max": 0.0,
@@ -4272,26 +4272,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2316884994506836,
- "epoch": 1.2439024390243902,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.35867181699723e-05,
- "kl": 1.1579370038816705e-05,
- "learning_rate": 5.856314121676467e-07,
- "loss": 5.79387460675207e-07,
- "num_tokens": 628224.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
"step": 153,
- "step_time": 4.375236807001784
+ "step_time": 22.82054339000024
},
{
"clip_ratio/high_max": 0.0,
@@ -4300,26 +4300,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4028943181037903,
- "epoch": 1.2520325203252032,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25418931245803833,
- "kl": 3.6890452065563295e-05,
- "learning_rate": 5.822703753824966e-07,
- "loss": 0.25599968433380127,
- "num_tokens": 631183.0,
- "reward": 0.38683533668518066,
- "reward_std": 0.43613559007644653,
- "rewards/true_env_reward_fn/mean": 0.38683533668518066,
- "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
"step": 154,
- "step_time": 6.055355972999678
+ "step_time": 9.973958625999785
},
{
"clip_ratio/high_max": 0.0,
@@ -4328,26 +4328,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.367663562297821,
- "epoch": 1.2601626016260163,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331811130046844,
- "kl": 1.2863993106293492e-05,
- "learning_rate": 5.788930179859024e-07,
- "loss": -0.0047044456005096436,
- "num_tokens": 636230.0,
- "reward": 0.3489508628845215,
- "reward_std": 0.13627417385578156,
- "rewards/true_env_reward_fn/mean": 0.3489508628845215,
- "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
"step": 155,
- "step_time": 3.9174396130001696
+ "step_time": 9.758407419000378
},
{
"clip_ratio/high_max": 0.0,
@@ -4356,26 +4356,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 135.0,
- "completions/max_terminated_length": 135.0,
- "completions/mean_length": 68.5,
- "completions/mean_terminated_length": 68.5,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.2759611010551453,
- "epoch": 1.2682926829268293,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12738144397735596,
- "kl": 1.5844128029129934e-05,
- "learning_rate": 5.754996423887061e-07,
- "loss": -0.011055335402488708,
- "num_tokens": 640262.0,
- "reward": 0.344404935836792,
- "reward_std": 0.16542991995811462,
- "rewards/true_env_reward_fn/mean": 0.344404935836792,
- "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
"step": 156,
- "step_time": 5.706334413998775
+ "step_time": 11.009583763999672
},
{
"clip_ratio/high_max": 0.0,
@@ -4384,26 +4384,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.0904476642608643,
- "epoch": 1.2764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.733699021628127e-05,
- "kl": 1.2238857834745431e-05,
- "learning_rate": 5.720905524360308e-07,
- "loss": 6.076299996493617e-07,
- "num_tokens": 645091.0,
- "reward": 0.4731999933719635,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4731999933719635,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
"step": 157,
- "step_time": 3.7696847109982627
+ "step_time": 10.12789283499933
},
{
"clip_ratio/high_max": 0.0,
@@ -4412,26 +4412,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.0,
- "completions/mean_terminated_length": 60.0,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.3856677412986755,
- "epoch": 1.2845528455284554,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18670028448104858,
- "kl": 1.8415606064081658e-05,
- "learning_rate": 5.686660533800736e-07,
- "loss": -0.07078710198402405,
- "num_tokens": 648179.0,
- "reward": 0.537517786026001,
- "reward_std": 0.1451217085123062,
- "rewards/true_env_reward_fn/mean": 0.537517786026001,
- "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
"step": 158,
- "step_time": 3.7075291149994882
+ "step_time": 7.800485663000472
},
{
"clip_ratio/high_max": 0.0,
@@ -4440,26 +4440,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.125,
- "completions/mean_terminated_length": 68.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.121916651725769,
- "epoch": 1.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11415883898735046,
- "kl": 1.909901220642496e-05,
- "learning_rate": 5.652264518527725e-07,
- "loss": -0.04401372745633125,
- "num_tokens": 652044.0,
- "reward": 0.5182899832725525,
- "reward_std": 0.21869486570358276,
- "rewards/true_env_reward_fn/mean": 0.5182899832725525,
- "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
"step": 159,
- "step_time": 3.8929355969994504
+ "step_time": 13.77649076500029
},
{
"clip_ratio/high_max": 0.0,
@@ -4468,26 +4468,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 143.0,
- "completions/max_terminated_length": 143.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5205118060112,
- "epoch": 1.3008130081300813,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.2403375506401062,
- "kl": 3.7574073758150917e-05,
- "learning_rate": 5.617720558383508e-07,
- "loss": 0.26385918259620667,
- "num_tokens": 656362.0,
- "reward": 0.41201668977737427,
- "reward_std": 0.2023741453886032,
- "rewards/true_env_reward_fn/mean": 0.41201668977737427,
- "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
"step": 160,
- "step_time": 6.157555950998358
+ "step_time": 13.25029361300085
},
{
"clip_ratio/high_max": 0.0,
@@ -4496,26 +4496,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3556928038597107,
- "epoch": 1.3089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18135924637317657,
- "kl": 2.6372636057203636e-05,
- "learning_rate": 5.583031746457407e-07,
- "loss": -0.10538280755281448,
- "num_tokens": 659977.0,
- "reward": 0.4239906072616577,
- "reward_std": 0.3287450969219208,
- "rewards/true_env_reward_fn/mean": 0.4239906072616577,
- "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
"step": 161,
- "step_time": 4.709477423999488
+ "step_time": 17.837881629000094
},
{
"clip_ratio/high_max": 0.0,
@@ -4524,26 +4524,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 52.5,
- "completions/mean_terminated_length": 52.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 0.9743769466876984,
- "epoch": 1.3170731707317074,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.178797647356987,
- "kl": 1.2532927030406427e-05,
- "learning_rate": 5.548201188808869e-07,
- "loss": -0.04164513945579529,
- "num_tokens": 661409.0,
- "reward": 0.8041956424713135,
- "reward_std": 0.1363772451877594,
- "rewards/true_env_reward_fn/mean": 0.8041956424713135,
- "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
"step": 162,
- "step_time": 3.1791253910014348
+ "step_time": 9.286757633000889
},
{
"clip_ratio/high_max": 0.0,
@@ -4552,26 +4552,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.3072250485420227,
- "epoch": 1.3252032520325203,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16768279671669006,
- "kl": 1.89386219062726e-05,
- "learning_rate": 5.513232004189339e-07,
- "loss": -0.01292814314365387,
- "num_tokens": 666504.0,
- "reward": 0.27981066703796387,
- "reward_std": 0.2949208915233612,
- "rewards/true_env_reward_fn/mean": 0.27981066703796387,
- "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
"step": 163,
- "step_time": 3.778431355000066
+ "step_time": 10.785562561999996
},
{
"clip_ratio/high_max": 0.0,
@@ -4580,26 +4580,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 51.0,
- "completions/mean_terminated_length": 51.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1984856128692627,
- "epoch": 1.3333333333333333,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13123764097690582,
- "kl": 1.9091786271019373e-05,
- "learning_rate": 5.478127323763027e-07,
- "loss": 0.035523779690265656,
- "num_tokens": 671004.0,
- "reward": 0.2771500051021576,
- "reward_std": 0.30146247148513794,
- "rewards/true_env_reward_fn/mean": 0.2771500051021576,
- "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
"step": 164,
- "step_time": 3.304021460000513
+ "step_time": 12.722446307000155
},
{
"clip_ratio/high_max": 0.0,
@@ -4608,26 +4608,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 102.0,
- "completions/max_terminated_length": 102.0,
- "completions/mean_length": 84.875,
- "completions/mean_terminated_length": 84.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2726752758026123,
- "epoch": 1.3414634146341464,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15594874322414398,
- "kl": 2.5981638827943243e-05,
- "learning_rate": 5.442890290826518e-07,
- "loss": -0.01398652046918869,
- "num_tokens": 677307.0,
- "reward": 0.35573017597198486,
- "reward_std": 0.25944042205810547,
- "rewards/true_env_reward_fn/mean": 0.35573017597198486,
- "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
"step": 165,
- "step_time": 4.972808451999299
+ "step_time": 14.89655208000022
},
{
"clip_ratio/high_max": 0.0,
@@ -4636,26 +4636,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 210.0,
- "completions/max_terminated_length": 210.0,
- "completions/mean_length": 103.625,
- "completions/mean_terminated_length": 103.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0949090719223022,
- "epoch": 1.3495934959349594,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.07191785424947739,
- "kl": 1.4828182884230046e-05,
- "learning_rate": 5.407524060527332e-07,
- "loss": -0.05063021928071976,
- "num_tokens": 682076.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
"step": 166,
- "step_time": 8.769379133000257
+ "step_time": 8.962532588000613
},
{
"clip_ratio/high_max": 0.0,
@@ -4664,26 +4664,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1857684254646301,
- "epoch": 1.3577235772357723,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.923542191041633e-05,
- "kl": 1.0165251751459436e-05,
- "learning_rate": 5.37203179958141e-07,
- "loss": 5.114516170579009e-07,
- "num_tokens": 685500.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
"step": 167,
- "step_time": 3.1906087530005607
+ "step_time": 11.708963983000103
},
{
"clip_ratio/high_max": 0.0,
@@ -4692,26 +4692,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 74.25,
- "completions/mean_terminated_length": 74.25,
- "completions/min_length": 16.0,
- "completions/min_terminated_length": 16.0,
- "entropy": 1.527149498462677,
- "epoch": 1.3658536585365852,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13164250552654266,
- "kl": 1.8541333702160046e-05,
- "learning_rate": 5.33641668598956e-07,
- "loss": -0.2347300797700882,
- "num_tokens": 688318.0,
- "reward": 0.7218117713928223,
- "reward_std": 0.1818692982196808,
- "rewards/true_env_reward_fn/mean": 0.7218117713928223,
- "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
"step": 168,
- "step_time": 8.50137474999974
+ "step_time": 16.905241970000134
},
{
"clip_ratio/high_max": 0.0,
@@ -4720,26 +4720,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 63.0,
- "completions/min_terminated_length": 63.0,
- "entropy": 1.3749513030052185,
- "epoch": 1.3739837398373984,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13246971368789673,
- "kl": 1.2620409506780561e-05,
- "learning_rate": 5.300681908752895e-07,
- "loss": 0.024534843862056732,
- "num_tokens": 692541.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
"step": 169,
- "step_time": 3.9512340759993094
+ "step_time": 12.27045077799994
},
{
"clip_ratio/high_max": 0.0,
@@ -4747,27 +4747,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 67.875,
- "completions/mean_terminated_length": 67.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2032299041748047,
- "epoch": 1.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11185794323682785,
- "kl": 1.5517784049734473e-05,
- "learning_rate": 5.264830667587295e-07,
- "loss": -0.05245225131511688,
- "num_tokens": 698064.0,
- "reward": 0.4444866180419922,
- "reward_std": 0.32400256395339966,
- "rewards/true_env_reward_fn/mean": 0.4444866180419922,
- "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
"step": 170,
- "step_time": 4.471538110999973
+ "step_time": 31.721865252000498
},
{
"clip_ratio/high_max": 0.0,
@@ -4776,26 +4776,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 95.0,
- "completions/mean_terminated_length": 95.0,
- "completions/min_length": 79.0,
- "completions/min_terminated_length": 79.0,
- "entropy": 1.4204387068748474,
- "epoch": 1.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1646534949541092,
- "kl": 2.4697198568901513e-05,
- "learning_rate": 5.228866172636899e-07,
- "loss": 0.02632315456867218,
- "num_tokens": 704196.0,
- "reward": 0.304565966129303,
- "reward_std": 0.32997164130210876,
- "rewards/true_env_reward_fn/mean": 0.304565966129303,
- "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
"step": 171,
- "step_time": 5.0436168590003945
+ "step_time": 8.973740739000277
},
{
"clip_ratio/high_max": 0.0,
@@ -4804,26 +4804,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1254178285598755,
- "epoch": 1.3983739837398375,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15058821439743042,
- "kl": 1.8407325114822015e-05,
- "learning_rate": 5.192791644186662e-07,
- "loss": 0.025478817522525787,
- "num_tokens": 706411.0,
- "reward": 0.7279239892959595,
- "reward_std": 0.11376125365495682,
- "rewards/true_env_reward_fn/mean": 0.7279239892959595,
- "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
"step": 172,
- "step_time": 3.2261944119982218
+ "step_time": 10.39117795600032
},
{
"clip_ratio/high_max": 0.0,
@@ -4832,26 +4832,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0894773602485657,
- "epoch": 1.4065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1687694787979126,
- "kl": 2.6046765015053097e-05,
- "learning_rate": 5.156610312374013e-07,
- "loss": -0.056941211223602295,
- "num_tokens": 711212.0,
- "reward": 0.4907146692276001,
- "reward_std": 0.3376546800136566,
- "rewards/true_env_reward_fn/mean": 0.4907146692276001,
- "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
"step": 173,
- "step_time": 3.9498180619993946
+ "step_time": 11.8111169230001
},
{
"clip_ratio/high_max": 0.0,
@@ -4860,26 +4860,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 60.125,
- "completions/mean_terminated_length": 60.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2238691449165344,
- "epoch": 1.4146341463414633,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19584441184997559,
- "kl": 4.8285241064149886e-05,
- "learning_rate": 5.120325416899629e-07,
- "loss": 0.0766875222325325,
- "num_tokens": 715409.0,
- "reward": 0.4593355059623718,
- "reward_std": 0.3909546136856079,
- "rewards/true_env_reward_fn/mean": 0.4593355059623718,
- "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
"step": 174,
- "step_time": 4.100519798999812
+ "step_time": 17.211099596999702
},
{
"clip_ratio/high_max": 0.0,
@@ -4888,26 +4888,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1763202548027039,
- "epoch": 1.4227642276422765,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1448555737733841,
- "kl": 1.2618989785551094e-05,
- "learning_rate": 5.08394020673734e-07,
- "loss": -0.012558378279209137,
- "num_tokens": 722327.0,
- "reward": 0.15966665744781494,
- "reward_std": 0.3235519230365753,
- "rewards/true_env_reward_fn/mean": 0.15966665744781494,
- "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
"step": 175,
- "step_time": 3.875348296000084
+ "step_time": 12.29185969800028
},
{
"clip_ratio/high_max": 0.0,
@@ -4916,26 +4916,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2852763533592224,
- "epoch": 1.4308943089430894,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22303813695907593,
- "kl": 4.580334098136518e-05,
- "learning_rate": 5.047457939843227e-07,
- "loss": -0.09214464575052261,
- "num_tokens": 726828.0,
- "reward": 0.25830498337745667,
- "reward_std": 0.37860655784606934,
- "rewards/true_env_reward_fn/mean": 0.25830498337745667,
- "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
"step": 176,
- "step_time": 4.241473076999682
+ "step_time": 10.987576109999736
},
{
"clip_ratio/high_max": 0.0,
@@ -4944,26 +4944,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.3609731197357178,
- "epoch": 1.4390243902439024,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.001985745271668e-05,
- "kl": 1.4942165307729738e-05,
- "learning_rate": 5.010881882863893e-07,
- "loss": 7.44550789022469e-07,
- "num_tokens": 729930.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
"step": 177,
- "step_time": 3.3902666960002534
+ "step_time": 10.792315139000493
},
{
"clip_ratio/high_max": 0.0,
@@ -4972,26 +4972,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2678966522216797,
- "epoch": 1.4471544715447155,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15351757407188416,
- "kl": 1.6737100395403104e-05,
- "learning_rate": 4.974215310843967e-07,
- "loss": 0.041131969541311264,
- "num_tokens": 732155.0,
- "reward": 0.7803820371627808,
- "reward_std": 0.08667682856321335,
- "rewards/true_env_reward_fn/mean": 0.7803820371627808,
- "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
"step": 178,
- "step_time": 3.637111981999624
+ "step_time": 12.399353334999887
},
{
"clip_ratio/high_max": 0.0,
@@ -5000,26 +5000,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.146271526813507,
- "epoch": 1.4552845528455285,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18466196954250336,
- "kl": 3.719841197380447e-05,
- "learning_rate": 4.937461506932859e-07,
- "loss": 0.029051154851913452,
- "num_tokens": 735418.0,
- "reward": 0.40377071499824524,
- "reward_std": 0.28345924615859985,
- "rewards/true_env_reward_fn/mean": 0.40377071499824524,
- "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
"step": 179,
- "step_time": 3.340555791999577
+ "step_time": 12.45691162600042
},
{
"clip_ratio/high_max": 0.0,
@@ -5028,26 +5028,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 170.0,
- "completions/max_terminated_length": 170.0,
- "completions/mean_length": 79.75,
- "completions/mean_terminated_length": 79.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.3902945518493652,
- "epoch": 1.4634146341463414,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.1020251139998436,
- "kl": 1.8220500351162627e-05,
- "learning_rate": 4.900623762090777e-07,
- "loss": -0.002344265580177307,
- "num_tokens": 740540.0,
- "reward": 0.3590222895145416,
- "reward_std": 0.12487777322530746,
- "rewards/true_env_reward_fn/mean": 0.3590222895145416,
- "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
"step": 180,
- "step_time": 7.219923718001155
+ "step_time": 12.108760526000424
},
{
"clip_ratio/high_max": 0.0,
@@ -5056,26 +5056,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.517557680606842,
- "epoch": 1.4715447154471546,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24192294478416443,
- "kl": 4.1268089262302965e-05,
- "learning_rate": 4.863705374794055e-07,
- "loss": 0.09132950007915497,
- "num_tokens": 744723.0,
- "reward": 0.23991000652313232,
- "reward_std": 0.2837013602256775,
- "rewards/true_env_reward_fn/mean": 0.23991000652313232,
- "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
"step": 181,
- "step_time": 3.696339096999509
+ "step_time": 14.16400909000049
},
{
"clip_ratio/high_max": 0.0,
@@ -5084,26 +5084,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.249614655971527,
- "epoch": 1.4796747967479675,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.1566745787858963,
- "kl": 2.6629099920683075e-05,
- "learning_rate": 4.826709650739811e-07,
- "loss": 0.003972277045249939,
- "num_tokens": 748979.0,
- "reward": 0.4935140311717987,
- "reward_std": 0.41978561878204346,
- "rewards/true_env_reward_fn/mean": 0.4935140311717987,
- "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
"step": 182,
- "step_time": 3.316512920000605
+ "step_time": 19.767916835999586
},
{
"clip_ratio/high_max": 0.0,
@@ -5112,26 +5112,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1666916608810425,
- "epoch": 1.4878048780487805,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001005820304271765,
- "kl": 1.2246940059412736e-05,
- "learning_rate": 4.789639902549948e-07,
- "loss": 6.330609494398232e-07,
- "num_tokens": 751323.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
"step": 183,
- "step_time": 3.7099916660008603
+ "step_time": 10.428195530999346
},
{
"clip_ratio/high_max": 0.0,
@@ -5140,26 +5140,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.3246222138404846,
- "epoch": 1.4959349593495934,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18140996992588043,
- "kl": 3.1042441150930244e-05,
- "learning_rate": 4.752499449474535e-07,
- "loss": -0.022353097796440125,
- "num_tokens": 755494.0,
- "reward": 0.4561777412891388,
- "reward_std": 0.2439236342906952,
- "rewards/true_env_reward_fn/mean": 0.4561777412891388,
- "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
"step": 184,
- "step_time": 3.7916486710000754
+ "step_time": 16.47171987999991
},
{
"clip_ratio/high_max": 0.0,
@@ -5168,26 +5168,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 75.75,
- "completions/mean_terminated_length": 75.75,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.2625707983970642,
- "epoch": 1.5040650406504064,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11702944338321686,
- "kl": 1.503958355897339e-05,
- "learning_rate": 4.715291617094607e-07,
- "loss": 0.023916304111480713,
- "num_tokens": 758432.0,
- "reward": 0.5015827417373657,
- "reward_std": 0.1783808320760727,
- "rewards/true_env_reward_fn/mean": 0.5015827417373657,
- "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
"step": 185,
- "step_time": 4.226409274000616
+ "step_time": 13.684267182999974
},
{
"clip_ratio/high_max": 0.0,
@@ -5196,26 +5196,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 54.0,
- "completions/mean_terminated_length": 54.0,
- "completions/min_length": 25.0,
- "completions/min_terminated_length": 25.0,
- "entropy": 1.1460023522377014,
- "epoch": 1.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11851620674133301,
- "kl": 1.3728345948038623e-05,
- "learning_rate": 4.678019737024387e-07,
- "loss": 0.0831337422132492,
- "num_tokens": 764336.0,
- "reward": 0.19094166159629822,
- "reward_std": 0.30934420228004456,
- "rewards/true_env_reward_fn/mean": 0.19094166159629822,
- "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
"step": 186,
- "step_time": 3.563357556000483
+ "step_time": 10.281018189999031
},
{
"clip_ratio/high_max": 0.0,
@@ -5224,26 +5224,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1291148662567139,
- "epoch": 1.5203252032520327,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.864256960805506e-05,
- "kl": 1.081683785741916e-05,
- "learning_rate": 4.6406871466129704e-07,
- "loss": 5.412177870312007e-07,
- "num_tokens": 766608.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
"step": 187,
- "step_time": 3.534869859002356
+ "step_time": 24.351223329000277
},
{
"clip_ratio/high_max": 0.0,
@@ -5252,26 +5252,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 92.0,
- "completions/mean_terminated_length": 92.0,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3645328283309937,
- "epoch": 1.5284552845528454,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18168245255947113,
- "kl": 2.6857565899263136e-05,
- "learning_rate": 4.6032971886454956e-07,
- "loss": -0.005156125873327255,
- "num_tokens": 774496.0,
- "reward": 0.11249999701976776,
- "reward_std": 0.20856082439422607,
- "rewards/true_env_reward_fn/mean": 0.11249999701976776,
- "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
"step": 188,
- "step_time": 8.34005261099992
+ "step_time": 10.644911742000204
},
{
"clip_ratio/high_max": 0.0,
@@ -5280,26 +5280,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1495982110500336,
- "epoch": 1.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14486820995807648,
- "kl": 1.3202762147557223e-05,
- "learning_rate": 4.5658532110438337e-07,
- "loss": -0.0010610297322273254,
- "num_tokens": 777186.0,
- "reward": 0.5879127383232117,
- "reward_std": 0.05142820253968239,
- "rewards/true_env_reward_fn/mean": 0.5879127383232117,
- "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
"step": 189,
- "step_time": 3.4856022139993
+ "step_time": 9.579594637999435
},
{
"clip_ratio/high_max": 0.0,
@@ -5308,26 +5308,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.241140365600586,
- "epoch": 1.5447154471544715,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001250067143701017,
- "kl": 1.5482702110602986e-05,
- "learning_rate": 4.52835856656681e-07,
- "loss": 7.80837922320643e-07,
- "num_tokens": 779965.0,
- "reward": 0.6861198544502258,
- "reward_std": 0.08807206153869629,
- "rewards/true_env_reward_fn/mean": 0.6861198544502258,
- "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
"step": 190,
- "step_time": 3.904181735999373
+ "step_time": 11.495368679999501
},
{
"clip_ratio/high_max": 0.0,
@@ -5336,26 +5336,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2944807410240173,
- "epoch": 1.5528455284552845,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2030111849308014,
- "kl": 3.585523518268019e-05,
- "learning_rate": 4.490816612509991e-07,
- "loss": 0.0143373291939497,
- "num_tokens": 786140.0,
- "reward": 0.35173332691192627,
- "reward_std": 0.18115806579589844,
- "rewards/true_env_reward_fn/mean": 0.35173332691192627,
- "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
"step": 191,
- "step_time": 4.005758510000305
+ "step_time": 7.909104242000012
},
{
"clip_ratio/high_max": 0.0,
@@ -5364,26 +5364,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 58.0,
- "completions/mean_terminated_length": 58.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2177271246910095,
- "epoch": 1.5609756097560976,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11188288033008575,
- "kl": 2.3622495064046234e-05,
- "learning_rate": 4.45323071040508e-07,
- "loss": -0.02369789034128189,
- "num_tokens": 790424.0,
- "reward": 0.4888629913330078,
- "reward_std": 0.23310808837413788,
- "rewards/true_env_reward_fn/mean": 0.4888629913330078,
- "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
"step": 192,
- "step_time": 3.2518814809991454
+ "step_time": 10.043768619000275
},
{
"clip_ratio/high_max": 0.0,
@@ -5392,26 +5392,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.192966103553772,
- "epoch": 1.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1378823071718216,
- "kl": 1.7358055174554465e-05,
- "learning_rate": 4.4156042257189143e-07,
- "loss": 0.06256310641765594,
- "num_tokens": 794521.0,
- "reward": 0.5219699740409851,
- "reward_std": 0.06214587390422821,
- "rewards/true_env_reward_fn/mean": 0.5219699740409851,
- "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
"step": 193,
- "step_time": 4.222739491999164
+ "step_time": 10.19648474899941
},
{
"clip_ratio/high_max": 0.0,
@@ -5420,26 +5420,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1770159006118774,
- "epoch": 1.5772357723577235,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19550059735774994,
- "kl": 2.4871268578863237e-05,
- "learning_rate": 4.377940527552125e-07,
- "loss": 0.05841376632452011,
- "num_tokens": 798194.0,
- "reward": 0.42302167415618896,
- "reward_std": 0.2911272346973419,
- "rewards/true_env_reward_fn/mean": 0.42302167415618896,
- "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
"step": 194,
- "step_time": 3.8170270639984665
+ "step_time": 14.372816425000565
},
{
"clip_ratio/high_max": 0.0,
@@ -5448,26 +5448,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.5244255661964417,
- "epoch": 1.5853658536585367,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1872202306985855,
- "kl": 2.5990483663917985e-05,
- "learning_rate": 4.340242988337462e-07,
- "loss": -0.044112429022789,
- "num_tokens": 802802.0,
- "reward": 0.33745431900024414,
- "reward_std": 0.22955451905727386,
- "rewards/true_env_reward_fn/mean": 0.33745431900024414,
- "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
"step": 195,
- "step_time": 4.27381555500142
+ "step_time": 11.282298853000157
},
{
"clip_ratio/high_max": 0.0,
@@ -5476,26 +5476,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.345891296863556,
- "epoch": 1.5934959349593496,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16891054809093475,
- "kl": 2.7261638024356216e-05,
- "learning_rate": 4.3025149835378275e-07,
- "loss": -0.139386385679245,
- "num_tokens": 807881.0,
- "reward": 0.3240283131599426,
- "reward_std": 0.2803676128387451,
- "rewards/true_env_reward_fn/mean": 0.3240283131599426,
- "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
"step": 196,
- "step_time": 4.981287381999209
+ "step_time": 8.65745804199969
},
{
"clip_ratio/high_max": 0.0,
@@ -5504,26 +5504,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.246802031993866,
- "epoch": 1.6016260162601625,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2096729576587677,
- "kl": 3.5958016269432846e-05,
- "learning_rate": 4.2647598913440264e-07,
- "loss": -0.02941281348466873,
- "num_tokens": 812500.0,
- "reward": 0.4126526415348053,
- "reward_std": 0.36393746733665466,
- "rewards/true_env_reward_fn/mean": 0.4126526415348053,
- "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
"step": 197,
- "step_time": 3.101726017999681
+ "step_time": 9.615110594999805
},
{
"clip_ratio/high_max": 0.0,
@@ -5532,26 +5532,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4369062185287476,
- "epoch": 1.6097560975609757,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1862105429172516,
- "kl": 4.7646244638599455e-05,
- "learning_rate": 4.2269810923722965e-07,
- "loss": 0.0521145761013031,
- "num_tokens": 818718.0,
- "reward": 0.26869943737983704,
- "reward_std": 0.1483483463525772,
- "rewards/true_env_reward_fn/mean": 0.26869943737983704,
- "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
"step": 198,
- "step_time": 4.287780451000799
+ "step_time": 10.832162847999825
},
{
"clip_ratio/high_max": 0.0,
@@ -5560,26 +5560,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 21.0,
- "completions/min_terminated_length": 21.0,
- "entropy": 1.192937195301056,
- "epoch": 1.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2013663798570633,
- "kl": 1.8760739294521045e-05,
- "learning_rate": 4.189181969361588e-07,
- "loss": 0.07236722111701965,
- "num_tokens": 825728.0,
- "reward": 0.23110000789165497,
- "reward_std": 0.23212090134620667,
- "rewards/true_env_reward_fn/mean": 0.23110000789165497,
- "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
"step": 199,
- "step_time": 4.708717262998107
+ "step_time": 10.94759418800004
},
{
"clip_ratio/high_max": 0.0,
@@ -5588,32 +5588,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.3792839050292969,
- "epoch": 1.6260162601626016,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.7985117463395e-05,
- "kl": 1.4280476534622721e-05,
- "learning_rate": 4.1513659068706814e-07,
- "loss": 7.153485626076872e-07,
- "num_tokens": 830318.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
"step": 200,
- "step_time": 3.9055351140013954
+ "step_time": 10.322844021000492
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 830318,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 4928134,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -5628,7 +5628,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-200/training_args.bin b/checkpoint-200/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-200/training_args.bin
+++ b/checkpoint-200/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-250/adapter_model.safetensors b/checkpoint-250/adapter_model.safetensors
index 436df24b34707027ac6d3abf5ca3b3f131bbd728..d5bc01b7636d15be9113eae0395ba383f04dceec 100644
--- a/checkpoint-250/adapter_model.safetensors
+++ b/checkpoint-250/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:778e2eb5058363271c6371ce49aa6d59c2c58ff759e8dc2c50f9f2c8378caa86
+oid sha256:74ce2462c180ca86c4d41f37d1844b51f23f6e0ca66435dd7d66be3e64d59369
size 8731128
diff --git a/checkpoint-250/optimizer.pt b/checkpoint-250/optimizer.pt
index 4ec03e7f1a862d34a2b4f789cdbfd8d6d406b92e..55f9445f4c9609640dbfc6f31465f1df30174d56 100644
--- a/checkpoint-250/optimizer.pt
+++ b/checkpoint-250/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:9f6c9d8b5c73cdba61371997fc1aa84a8aa321b3b3587dfbdaa6c33b016269d3
+oid sha256:886900e0c6349e49a4aa05cd49f213fa97285306851251fc61571fab25e0f0c2
size 17526842
diff --git a/checkpoint-250/ref/adapter_model.safetensors b/checkpoint-250/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-250/ref/adapter_model.safetensors
+++ b/checkpoint-250/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-250/rng_state.pth b/checkpoint-250/rng_state.pth
index b165d5b908fe58578b31b49bfb858eeb56018301..f051508182571d9a872d4e565f2c5769a58fa515 100644
--- a/checkpoint-250/rng_state.pth
+++ b/checkpoint-250/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:6117437beb154fdbe42da769bc751661179f0830057abcaeb1073a915d11a232
+oid sha256:893f9a4dc8e0a96455069270a12e78290237825df7bda80d1e36733236d60e65
size 14244
diff --git a/checkpoint-250/scheduler.pt b/checkpoint-250/scheduler.pt
index a3be729a321146873c53daa0d0e4d870636afc87..727e7c632a88e58f0b0299efa59f5c49bb1494fb 100644
--- a/checkpoint-250/scheduler.pt
+++ b/checkpoint-250/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:40db5c7e3b57ff5ba822a7ba8434e7a868d756b70556c6f711c44f3bda3fde48
+oid sha256:099fe8a4bdabd76b89aa66c11a875f9a0d731f355db90e5ed7e0828b88cf7c25
size 1064
diff --git a/checkpoint-250/trainer_state.json b/checkpoint-250/trainer_state.json
index c91fac69447982f13bbebe56ba80c2e8f86ade45..b8ae99693a93418d8e0bc6281764353e667b9b50 100644
--- a/checkpoint-250/trainer_state.json
+++ b/checkpoint-250/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 2.032520325203252,
+ "epoch": 6.097560975609756,
"eval_steps": 500,
"global_step": 250,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,54 +1668,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
- "clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,54 +2648,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
},
{
"clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.0,
- "completions/mean_terminated_length": 59.0,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2509461045265198,
- "epoch": 0.8211382113821138,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22887632250785828,
- "kl": 2.1612477212329395e-05,
- "learning_rate": 7.310023360571047e-07,
- "loss": 0.025605827569961548,
- "num_tokens": 414080.0,
- "reward": 0.588032603263855,
- "reward_std": 0.11032751202583313,
- "rewards/true_env_reward_fn/mean": 0.588032603263855,
- "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
"step": 101,
- "step_time": 3.625197022998691
+ "step_time": 16.478299477000064
},
{
"clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 176.0,
- "completions/max_terminated_length": 176.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5443179607391357,
- "epoch": 0.8292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10614532232284546,
- "kl": 1.492139062975184e-05,
- "learning_rate": 7.28862404677924e-07,
- "loss": 0.06531564146280289,
- "num_tokens": 419835.0,
- "reward": 0.07074306160211563,
- "reward_std": 0.2918013632297516,
- "rewards/true_env_reward_fn/mean": 0.07074306160211563,
- "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
"step": 102,
- "step_time": 7.796810614998321
+ "step_time": 11.344593008000174
},
{
"clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 101.0,
"completions/max_terminated_length": 101.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3223788738250732,
- "epoch": 0.8373983739837398,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2421368807554245,
- "kl": 3.265505938543356e-05,
- "learning_rate": 7.266930267388503e-07,
- "loss": -0.07752113044261932,
- "num_tokens": 422773.0,
- "reward": 0.33568501472473145,
- "reward_std": 0.2780380845069885,
- "rewards/true_env_reward_fn/mean": 0.33568501472473145,
- "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
"step": 103,
- "step_time": 4.313938073000827
+ "step_time": 8.135681302999728
},
{
"clip_ratio/high_max": 0.0,
@@ -2900,26 +2900,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 67.25,
- "completions/mean_terminated_length": 67.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.3332037329673767,
- "epoch": 0.8455284552845529,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13883370161056519,
- "kl": 2.1224042484391248e-05,
- "learning_rate": 7.244943964874369e-07,
- "loss": 0.021739646792411804,
- "num_tokens": 426507.0,
- "reward": 0.40595096349716187,
- "reward_std": 0.2035457342863083,
- "rewards/true_env_reward_fn/mean": 0.40595096349716187,
- "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
"step": 104,
- "step_time": 4.155937195999286
+ "step_time": 15.185034105999875
},
{
"clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
"completions/min_length": 31.0,
"completions/min_terminated_length": 31.0,
- "entropy": 1.3391229510307312,
- "epoch": 0.8536585365853658,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24205009639263153,
- "kl": 2.5022183763212524e-05,
- "learning_rate": 7.222667107905085e-07,
- "loss": 0.06330433487892151,
- "num_tokens": 429010.0,
- "reward": 0.3355163037776947,
- "reward_std": 0.2902730703353882,
- "rewards/true_env_reward_fn/mean": 0.3355163037776947,
- "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
"step": 105,
- "step_time": 3.808478789000219
+ "step_time": 9.588520330999927
},
{
"clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1864720582962036,
- "epoch": 0.8617886178861789,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13473568856716156,
- "kl": 1.4212585938366828e-05,
- "learning_rate": 7.200101691165338e-07,
- "loss": -0.020715661346912384,
- "num_tokens": 432403.0,
- "reward": 0.4871198534965515,
- "reward_std": 0.15407639741897583,
- "rewards/true_env_reward_fn/mean": 0.4871198534965515,
- "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
"step": 106,
- "step_time": 4.240638332001254
+ "step_time": 14.60399662399982
},
{
"clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.0,
- "completions/mean_terminated_length": 48.0,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "entropy": 1.0669284462928772,
- "epoch": 0.8699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14131899178028107,
- "kl": 1.5787159554747632e-05,
- "learning_rate": 7.177249735177651e-07,
- "loss": 0.03678784519433975,
- "num_tokens": 435995.0,
- "reward": 0.5010770559310913,
- "reward_std": 0.48966261744499207,
- "rewards/true_env_reward_fn/mean": 0.5010770559310913,
- "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
"step": 107,
- "step_time": 3.3587191269998584
+ "step_time": 20.100954443999854
},
{
"clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 71.875,
- "completions/mean_terminated_length": 71.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.304731547832489,
- "epoch": 0.8780487804878049,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.380985673284158e-05,
- "kl": 1.3128728824085556e-05,
- "learning_rate": 7.154113286121462e-07,
- "loss": 6.494262834166875e-07,
- "num_tokens": 442094.0,
- "reward": 0.4055500030517578,
- "reward_std": 0.052258480340242386,
- "rewards/true_env_reward_fn/mean": 0.4055500030517578,
- "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
"step": 108,
- "step_time": 4.337008413998774
+ "step_time": 8.304149297999857
},
{
"clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2324069738388062,
- "epoch": 0.8861788617886179,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13588950037956238,
- "kl": 1.3448377558233915e-05,
- "learning_rate": 7.130694415649912e-07,
- "loss": 1.0952353477478027e-06,
- "num_tokens": 447226.0,
- "reward": 0.20854972302913666,
- "reward_std": 0.06059705466032028,
- "rewards/true_env_reward_fn/mean": 0.20854972302913666,
- "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
"step": 109,
- "step_time": 3.2976038649994734
+ "step_time": 11.260021517000041
},
{
"clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1567262411117554,
- "epoch": 0.8943089430894309,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.736967720324174e-05,
- "kl": 1.2838129805459175e-05,
- "learning_rate": 7.106995220704342e-07,
- "loss": 6.425898391171359e-07,
- "num_tokens": 450359.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
"step": 110,
- "step_time": 4.067084037998939
+ "step_time": 22.80178854000019
},
{
"clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 100.0,
"completions/max_terminated_length": 100.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 12.0,
- "completions/min_terminated_length": 12.0,
- "entropy": 1.496058464050293,
- "epoch": 0.9024390243902439,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18041981756687164,
- "kl": 1.6616825632809196e-05,
- "learning_rate": 7.083017823326532e-07,
- "loss": 0.0269068144261837,
- "num_tokens": 453583.0,
- "reward": 0.5647265911102295,
- "reward_std": 0.1507105529308319,
- "rewards/true_env_reward_fn/mean": 0.5647265911102295,
- "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
"step": 111,
- "step_time": 4.347732382997492
+ "step_time": 9.663163859000178
},
{
"clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 67.75,
- "completions/mean_terminated_length": 67.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.4196155667304993,
- "epoch": 0.9105691056910569,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18451112508773804,
- "kl": 2.1803500203532167e-05,
- "learning_rate": 7.058764370468698e-07,
- "loss": 0.1650262475013733,
- "num_tokens": 456773.0,
- "reward": 0.6907394528388977,
- "reward_std": 0.1393815129995346,
- "rewards/true_env_reward_fn/mean": 0.6907394528388977,
- "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
"step": 112,
- "step_time": 4.627644968999448
+ "step_time": 9.903081222999617
},
{
"clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2243221998214722,
- "epoch": 0.9186991869918699,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331371545791626,
- "kl": 1.548633599668392e-05,
- "learning_rate": 7.034237033801247e-07,
- "loss": 0.039844345301389694,
- "num_tokens": 462714.0,
- "reward": 0.21676866710186005,
- "reward_std": 0.26559779047966003,
- "rewards/true_env_reward_fn/mean": 0.21676866710186005,
- "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
"step": 113,
- "step_time": 3.8455466220002563
+ "step_time": 10.101770388999284
},
{
"clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 73.0,
- "completions/mean_terminated_length": 73.0,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.325823724269867,
- "epoch": 0.926829268292683,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1866220384836197,
- "kl": 1.8801019905367866e-05,
- "learning_rate": 7.009438009518325e-07,
- "loss": 0.06504581868648529,
- "num_tokens": 465994.0,
- "reward": 0.5194582939147949,
- "reward_std": 0.2796703577041626,
- "rewards/true_env_reward_fn/mean": 0.5194582939147949,
- "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
"step": 114,
- "step_time": 4.151028698999653
+ "step_time": 17.633509853000305
},
{
"clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1521879434585571,
- "epoch": 0.9349593495934959,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14320029318332672,
- "kl": 1.2749982033710694e-05,
- "learning_rate": 6.98436951814117e-07,
- "loss": 0.03685300797224045,
- "num_tokens": 468615.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
"step": 115,
- "step_time": 3.6973990600017714
+ "step_time": 12.307247350000125
},
{
"clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.201507806777954,
- "epoch": 0.943089430894309,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14264807105064392,
- "kl": 2.6679515940486453e-05,
- "learning_rate": 6.959033804319283e-07,
- "loss": -0.023484818637371063,
- "num_tokens": 471647.0,
- "reward": 0.41836902499198914,
- "reward_std": 0.3116860091686249,
- "rewards/true_env_reward_fn/mean": 0.41836902499198914,
- "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
"step": 116,
- "step_time": 3.1295652919998247
+ "step_time": 14.468690254000194
},
{
"clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 62.375,
- "completions/mean_terminated_length": 62.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2834057807922363,
- "epoch": 0.9512195121951219,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.810227154754102e-05,
- "kl": 1.4841665233689127e-05,
- "learning_rate": 6.933433136629443e-07,
- "loss": 7.425555850204546e-07,
- "num_tokens": 474682.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
"step": 117,
- "step_time": 3.4368692790012574
+ "step_time": 11.849063975999798
},
{
"clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.191932499408722,
- "epoch": 0.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21204856038093567,
- "kl": 3.64198385796044e-05,
- "learning_rate": 6.907569807372574e-07,
- "loss": -0.001312553882598877,
- "num_tokens": 477027.0,
- "reward": 0.5300568342208862,
- "reward_std": 0.2945883274078369,
- "rewards/true_env_reward_fn/mean": 0.5300568342208862,
- "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
"step": 118,
- "step_time": 3.8569856240010267
+ "step_time": 16.597334930000216
},
{
"clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 190.0,
- "completions/max_terminated_length": 190.0,
- "completions/mean_length": 96.5,
- "completions/mean_terminated_length": 96.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2401175498962402,
- "epoch": 0.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011377666669432074,
- "kl": 1.3742283954343293e-05,
- "learning_rate": 6.881446132368494e-07,
- "loss": 6.866695230201003e-07,
- "num_tokens": 481999.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
"step": 119,
- "step_time": 8.09440958399864
+ "step_time": 11.729475523999554
},
{
"clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.230682611465454,
- "epoch": 0.975609756097561,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22175048291683197,
- "kl": 1.2522132237791084e-05,
- "learning_rate": 6.855064450748555e-07,
- "loss": -0.04083740711212158,
- "num_tokens": 490884.0,
- "reward": 0.13476666808128357,
- "reward_std": 0.2987530529499054,
- "rewards/true_env_reward_fn/mean": 0.13476666808128357,
- "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
"step": 120,
- "step_time": 4.678523641001448
+ "step_time": 11.503627788000358
},
{
"clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2799639105796814,
- "epoch": 0.983739837398374,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19926966726779938,
- "kl": 1.7022688552970067e-05,
- "learning_rate": 6.828427124746189e-07,
- "loss": -0.010804429650306702,
- "num_tokens": 496404.0,
- "reward": 0.24633333086967468,
- "reward_std": 0.2454334795475006,
- "rewards/true_env_reward_fn/mean": 0.24633333086967468,
- "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
"step": 121,
- "step_time": 3.98071062300005
+ "step_time": 8.728293746000418
},
{
"clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 46.875,
- "completions/mean_terminated_length": 46.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3840235471725464,
- "epoch": 0.991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24853822588920593,
- "kl": 3.688259130285587e-05,
- "learning_rate": 6.801536539485403e-07,
- "loss": 0.10205884277820587,
- "num_tokens": 499767.0,
- "reward": 0.3045905530452728,
- "reward_std": 0.262839138507843,
- "rewards/true_env_reward_fn/mean": 0.3045905530452728,
- "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
"step": 122,
- "step_time": 3.3792565210005705
+ "step_time": 15.51957702100026
},
{
"clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 50.375,
- "completions/mean_terminated_length": 50.375,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2064164280891418,
- "epoch": 1.0,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1365528553724289,
- "kl": 2.434901080050622e-05,
- "learning_rate": 6.774395102767203e-07,
- "loss": -0.03472680225968361,
- "num_tokens": 504906.0,
- "reward": 0.2722649872303009,
- "reward_std": 0.2922348082065582,
- "rewards/true_env_reward_fn/mean": 0.2722649872303009,
- "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
"step": 123,
- "step_time": 3.0233660449994204
+ "step_time": 8.471463828000196
},
{
"clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2819936871528625,
- "epoch": 1.008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00023045104171615094,
- "kl": 2.2608143808611203e-05,
- "learning_rate": 6.747005244854004e-07,
- "loss": 1.1284330412308918e-06,
- "num_tokens": 508329.0,
- "reward": 0.3149532079696655,
- "reward_std": 0.4275679290294647,
- "rewards/true_env_reward_fn/mean": 0.3149532079696655,
- "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
"step": 124,
- "step_time": 4.01701365199915
+ "step_time": 13.438758649999727
},
{
"clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 157.0,
- "completions/max_terminated_length": 157.0,
- "completions/mean_length": 80.875,
- "completions/mean_terminated_length": 80.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1542360186576843,
- "epoch": 1.016260162601626,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.232049003709108e-05,
- "kl": 1.291002809011843e-05,
- "learning_rate": 6.719369418252023e-07,
- "loss": 6.488799613180163e-07,
- "num_tokens": 515076.0,
- "reward": 0.4841846525669098,
- "reward_std": 0.12780573964118958,
- "rewards/true_env_reward_fn/mean": 0.4841846525669098,
- "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
"step": 125,
- "step_time": 7.240956699999515
+ "step_time": 8.654177170999901
},
{
"clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2594389915466309,
- "epoch": 1.024390243902439,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13091468811035156,
- "kl": 1.751603304001037e-05,
- "learning_rate": 6.691490097491675e-07,
- "loss": -0.033413223922252655,
- "num_tokens": 520279.0,
- "reward": 0.28095200657844543,
- "reward_std": 0.21837711334228516,
- "rewards/true_env_reward_fn/mean": 0.28095200657844543,
- "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
"step": 126,
- "step_time": 3.355879656997786
+ "step_time": 10.513378469000145
},
{
"clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 136.0,
- "completions/max_terminated_length": 136.0,
- "completions/mean_length": 77.875,
- "completions/mean_terminated_length": 77.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.3044686317443848,
- "epoch": 1.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12953205406665802,
- "kl": 1.8700401597016025e-05,
- "learning_rate": 6.663369778906008e-07,
- "loss": 0.03562816232442856,
- "num_tokens": 524582.0,
- "reward": 0.4330660402774811,
- "reward_std": 0.4592672288417816,
- "rewards/true_env_reward_fn/mean": 0.4330660402774811,
- "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
"step": 127,
- "step_time": 5.965807722999671
+ "step_time": 23.20779430600078
},
{
"clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 117.0,
- "completions/max_terminated_length": 117.0,
- "completions/mean_length": 75.375,
- "completions/mean_terminated_length": 75.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1742327809333801,
- "epoch": 1.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14993594586849213,
- "kl": 1.1459212601039326e-05,
- "learning_rate": 6.635010980407174e-07,
- "loss": 0.03646668791770935,
- "num_tokens": 526213.0,
- "reward": 0.7185037136077881,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.7185037136077881,
- "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
"step": 128,
- "step_time": 4.9305356690001645
+ "step_time": 11.551069149999876
},
{
"clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 49.125,
- "completions/mean_terminated_length": 49.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0784690976142883,
- "epoch": 1.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16007214784622192,
- "kl": 1.2491957932070363e-05,
- "learning_rate": 6.606416241260979e-07,
- "loss": 0.006608985364437103,
- "num_tokens": 531862.0,
- "reward": 0.2934249937534332,
- "reward_std": 0.2395382523536682,
- "rewards/true_env_reward_fn/mean": 0.2934249937534332,
- "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
"step": 129,
- "step_time": 3.173622508000335
+ "step_time": 10.723005456000465
},
{
"clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.308219850063324,
- "epoch": 1.056910569105691,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.985446427483112e-05,
- "kl": 1.2420873190421844e-05,
- "learning_rate": 6.577588121859508e-07,
- "loss": 6.241918413252279e-07,
- "num_tokens": 535957.0,
- "reward": 0.4817493259906769,
- "reward_std": 0.029202036559581757,
- "rewards/true_env_reward_fn/mean": 0.4817493259906769,
- "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
"step": 130,
- "step_time": 4.251137947001553
+ "step_time": 15.15810051499966
},
{
"clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.0767641067504883,
- "epoch": 1.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15786457061767578,
- "kl": 1.8847958926926367e-05,
- "learning_rate": 6.548529203491875e-07,
- "loss": -0.0026272237300872803,
- "num_tokens": 539269.0,
- "reward": 0.536803662776947,
- "reward_std": 0.30375123023986816,
- "rewards/true_env_reward_fn/mean": 0.536803662776947,
- "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
"step": 131,
- "step_time": 3.7980547870010923
+ "step_time": 20.109428818001106
},
{
"clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3295028805732727,
- "epoch": 1.0731707317073171,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1881481558084488,
- "kl": 2.0969039724150207e-05,
- "learning_rate": 6.519242088113085e-07,
- "loss": 0.08431969583034515,
- "num_tokens": 545691.0,
- "reward": 0.24590599536895752,
- "reward_std": 0.2047487199306488,
- "rewards/true_env_reward_fn/mean": 0.24590599536895752,
- "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
"step": 132,
- "step_time": 4.361092664001262
+ "step_time": 10.12403799699996
},
{
"clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2131375670433044,
- "epoch": 1.08130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13930389285087585,
- "kl": 1.1046585314034019e-05,
- "learning_rate": 6.489729398111058e-07,
- "loss": -0.03801802545785904,
- "num_tokens": 550295.0,
- "reward": 0.3215479254722595,
- "reward_std": 0.1736886352300644,
- "rewards/true_env_reward_fn/mean": 0.3215479254722595,
- "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
"step": 133,
- "step_time": 3.372364626999115
+ "step_time": 10.396350653999434
},
{
"clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 63.375,
- "completions/mean_terminated_length": 63.375,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2786019444465637,
- "epoch": 1.089430894308943,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12932609021663666,
- "kl": 1.340499647994875e-05,
- "learning_rate": 6.459993776071815e-07,
- "loss": 0.029022663831710815,
- "num_tokens": 553826.0,
- "reward": 0.4830188751220703,
- "reward_std": 0.29014864563941956,
- "rewards/true_env_reward_fn/mean": 0.4830188751220703,
- "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
"step": 134,
- "step_time": 3.215292060998763
+ "step_time": 10.961974539000039
},
{
"clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 0.9003906548023224,
- "epoch": 1.0975609756097562,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15958240628242493,
- "kl": 2.838099044311093e-05,
- "learning_rate": 6.430037884542861e-07,
- "loss": 0.11459673941135406,
- "num_tokens": 557217.0,
- "reward": 0.494448184967041,
- "reward_std": 0.3076546788215637,
- "rewards/true_env_reward_fn/mean": 0.494448184967041,
- "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
"step": 135,
- "step_time": 3.500462582000182
+ "step_time": 11.779171687000144
},
{
"clip_ratio/high_max": 0.0,
@@ -3797,25 +3797,25 @@
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
"completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2928712964057922,
- "epoch": 1.1056910569105691,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23077522218227386,
- "kl": 2.251418845844455e-05,
- "learning_rate": 6.399864405794782e-07,
- "loss": -0.05874824523925781,
- "num_tokens": 562421.0,
- "reward": 0.2385583370923996,
- "reward_std": 0.23380905389785767,
- "rewards/true_env_reward_fn/mean": 0.2385583370923996,
- "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
"step": 136,
- "step_time": 4.208805245998519
+ "step_time": 8.779588141000204
},
{
"clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8736326098442078,
- "epoch": 1.113821138211382,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.871674071997404e-05,
- "kl": 1.1485328741400735e-05,
- "learning_rate": 6.369476041581066e-07,
- "loss": 5.747077125306532e-07,
- "num_tokens": 566387.0,
- "reward": 0.4902166724205017,
- "reward_std": 0.038254011422395706,
- "rewards/true_env_reward_fn/mean": 0.4902166724205017,
- "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
"step": 137,
- "step_time": 3.981489739000608
+ "step_time": 11.74765996799988
},
{
"clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1538971662521362,
- "epoch": 1.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14676779508590698,
- "kl": 1.1651037766569061e-05,
- "learning_rate": 6.338875512896188e-07,
- "loss": 0.1347643882036209,
- "num_tokens": 569341.0,
- "reward": 0.43844783306121826,
- "reward_std": 0.16067014634609222,
- "rewards/true_env_reward_fn/mean": 0.43844783306121826,
- "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
"step": 138,
- "step_time": 3.689221037999232
+ "step_time": 8.8135579650002
},
{
"clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1961707472801208,
- "epoch": 1.1300813008130082,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16650564968585968,
- "kl": 1.4349476259667426e-05,
- "learning_rate": 6.308065559731976e-07,
- "loss": 0.007910434156656265,
- "num_tokens": 574046.0,
- "reward": 0.4596000015735626,
- "reward_std": 0.07715634256601334,
- "rewards/true_env_reward_fn/mean": 0.4596000015735626,
- "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
"step": 139,
- "step_time": 3.6711935700004688
+ "step_time": 9.997661417000472
},
{
"clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2013322114944458,
- "epoch": 1.1382113821138211,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14415834844112396,
- "kl": 1.4664098671346437e-05,
- "learning_rate": 6.277048940832264e-07,
- "loss": -0.016162052750587463,
- "num_tokens": 576769.0,
- "reward": 0.6152583360671997,
- "reward_std": 0.07727260142564774,
- "rewards/true_env_reward_fn/mean": 0.6152583360671997,
- "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
"step": 140,
- "step_time": 3.5191362610003125
+ "step_time": 14.679971276999822
},
{
"clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.75,
- "completions/mean_terminated_length": 44.75,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0287770330905914,
- "epoch": 1.146341463414634,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.892051457427442e-05,
- "kl": 1.1797974821092794e-05,
- "learning_rate": 6.245828433445872e-07,
- "loss": 5.92092192164273e-07,
- "num_tokens": 578843.0,
- "reward": 0.6387845277786255,
- "reward_std": 0.13867565989494324,
- "rewards/true_env_reward_fn/mean": 0.6387845277786255,
- "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
"step": 141,
- "step_time": 2.5015027329991426
+ "step_time": 11.64747691499997
},
{
"clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1590029001235962,
- "epoch": 1.1544715447154472,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.160966694355011,
- "kl": 1.4735675904375967e-05,
- "learning_rate": 6.214406833077937e-07,
- "loss": 0.0170527845621109,
- "num_tokens": 583201.0,
- "reward": 0.36017733812332153,
- "reward_std": 0.3556094467639923,
- "rewards/true_env_reward_fn/mean": 0.36017733812332153,
- "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
"step": 142,
- "step_time": 3.2783409929998015
+ "step_time": 8.380270293999729
},
{
"clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 59.0,
- "completions/min_terminated_length": 59.0,
- "entropy": 1.1985241174697876,
- "epoch": 1.1626016260162602,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12272457778453827,
- "kl": 1.7849098185251933e-05,
- "learning_rate": 6.182786953239593e-07,
- "loss": -0.0016125142574310303,
- "num_tokens": 587317.0,
- "reward": 0.34745320677757263,
- "reward_std": 0.3954337239265442,
- "rewards/true_env_reward_fn/mean": 0.34745320677757263,
- "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
"step": 143,
- "step_time": 3.9932043310000154
+ "step_time": 11.716556537000088
},
{
"clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 48.0,
- "completions/max_terminated_length": 48.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1116944551467896,
- "epoch": 1.170731707317073,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18435250222682953,
- "kl": 1.014559029499651e-05,
- "learning_rate": 6.150971625196048e-07,
- "loss": 0.009793907403945923,
- "num_tokens": 590191.0,
- "reward": 0.4938516616821289,
- "reward_std": 0.03703190013766289,
- "rewards/true_env_reward_fn/mean": 0.4938516616821289,
- "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
"step": 144,
- "step_time": 2.3663663690022076
+ "step_time": 10.67718802499985
},
{
"clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.0,
- "completions/mean_terminated_length": 53.0,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.413200855255127,
- "epoch": 1.1788617886178863,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14336225390434265,
- "kl": 2.0541991034406237e-05,
- "learning_rate": 6.118963697713078e-07,
- "loss": -0.013927727937698364,
- "num_tokens": 593671.0,
- "reward": 0.4619143605232239,
- "reward_std": 0.3773181140422821,
- "rewards/true_env_reward_fn/mean": 0.4619143605232239,
- "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
"step": 145,
- "step_time": 3.9730388410007436
+ "step_time": 8.610043666999445
},
{
"clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 128.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 6.0,
- "completions/min_terminated_length": 6.0,
- "entropy": 1.08676016330719,
- "epoch": 1.1869918699186992,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15555191040039062,
- "kl": 1.6947700260061538e-05,
- "learning_rate": 6.086766036801937e-07,
- "loss": -0.139797180891037,
- "num_tokens": 601612.0,
- "reward": 0.3831036686897278,
- "reward_std": 0.09242849797010422,
- "rewards/true_env_reward_fn/mean": 0.3831036686897278,
- "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
"step": 146,
- "step_time": 6.323679949000507
+ "step_time": 13.649344002000362
},
{
"clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.5055813789367676,
- "epoch": 1.1951219512195121,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2397669553756714,
- "kl": 2.704876442294335e-05,
- "learning_rate": 6.054381525462745e-07,
- "loss": 0.2738838493824005,
- "num_tokens": 606712.0,
- "reward": 0.25339600443840027,
- "reward_std": 0.3023079037666321,
- "rewards/true_env_reward_fn/mean": 0.25339600443840027,
- "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
"step": 147,
- "step_time": 5.185072233998653
+ "step_time": 11.702765863000877
},
{
"clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 47.25,
- "completions/mean_terminated_length": 47.25,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.135968267917633,
- "epoch": 1.203252032520325,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2614514231681824,
- "kl": 3.613240005506668e-05,
- "learning_rate": 6.021813063426323e-07,
- "loss": 0.10286401212215424,
- "num_tokens": 610566.0,
- "reward": 0.31031692028045654,
- "reward_std": 0.3124054968357086,
- "rewards/true_env_reward_fn/mean": 0.31031692028045654,
- "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
"step": 148,
- "step_time": 3.2177847610000754
+ "step_time": 13.306644664000487
},
{
"clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 52.75,
- "completions/mean_terminated_length": 52.75,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4589928984642029,
- "epoch": 1.2113821138211383,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2019941508769989,
- "kl": 2.1841721718374174e-05,
- "learning_rate": 5.989063566894572e-07,
- "loss": 0.010915875434875488,
- "num_tokens": 615716.0,
- "reward": 0.31711751222610474,
- "reward_std": 0.13289952278137207,
- "rewards/true_env_reward_fn/mean": 0.31711751222610474,
- "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
"step": 149,
- "step_time": 4.3804878079990885
+ "step_time": 9.446422488999815
},
{
"clip_ratio/high_max": 0.0,
@@ -4188,26 +4188,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.1892729997634888,
- "epoch": 1.2195121951219512,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.835455471649766e-05,
- "kl": 1.3420096820482286e-05,
- "learning_rate": 5.956135968279332e-07,
- "loss": 6.646802717114042e-07,
- "num_tokens": 619439.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
"step": 150,
- "step_time": 3.63938895299907
+ "step_time": 10.538116119999813
},
{
"clip_ratio/high_max": 0.0,
@@ -4216,26 +4216,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1402934789657593,
- "epoch": 1.2276422764227641,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010795716661959887,
- "kl": 1.4652535810455447e-05,
- "learning_rate": 5.923033215939834e-07,
- "loss": 7.542968205598299e-07,
- "num_tokens": 621009.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
"step": 151,
- "step_time": 3.926544339999964
+ "step_time": 15.755764130999978
},
{
"clip_ratio/high_max": 0.0,
@@ -4244,26 +4244,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
"completions/mean_length": 64.125,
"completions/mean_terminated_length": 64.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.190350890159607,
- "epoch": 1.2357723577235773,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19853363931179047,
- "kl": 2.269768037876929e-05,
- "learning_rate": 5.889758273918683e-07,
- "loss": 0.044217392802238464,
- "num_tokens": 623994.0,
- "reward": 0.4411996603012085,
- "reward_std": 0.2517909109592438,
- "rewards/true_env_reward_fn/mean": 0.4411996603012085,
- "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
"step": 152,
- "step_time": 3.7339736520007136
+ "step_time": 9.670861957999932
},
{
"clip_ratio/high_max": 0.0,
@@ -4272,26 +4272,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2316884994506836,
- "epoch": 1.2439024390243902,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.35867181699723e-05,
- "kl": 1.1579370038816705e-05,
- "learning_rate": 5.856314121676467e-07,
- "loss": 5.79387460675207e-07,
- "num_tokens": 628224.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
"step": 153,
- "step_time": 4.375236807001784
+ "step_time": 22.82054339000024
},
{
"clip_ratio/high_max": 0.0,
@@ -4300,26 +4300,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4028943181037903,
- "epoch": 1.2520325203252032,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25418931245803833,
- "kl": 3.6890452065563295e-05,
- "learning_rate": 5.822703753824966e-07,
- "loss": 0.25599968433380127,
- "num_tokens": 631183.0,
- "reward": 0.38683533668518066,
- "reward_std": 0.43613559007644653,
- "rewards/true_env_reward_fn/mean": 0.38683533668518066,
- "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
"step": 154,
- "step_time": 6.055355972999678
+ "step_time": 9.973958625999785
},
{
"clip_ratio/high_max": 0.0,
@@ -4328,26 +4328,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.367663562297821,
- "epoch": 1.2601626016260163,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331811130046844,
- "kl": 1.2863993106293492e-05,
- "learning_rate": 5.788930179859024e-07,
- "loss": -0.0047044456005096436,
- "num_tokens": 636230.0,
- "reward": 0.3489508628845215,
- "reward_std": 0.13627417385578156,
- "rewards/true_env_reward_fn/mean": 0.3489508628845215,
- "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
"step": 155,
- "step_time": 3.9174396130001696
+ "step_time": 9.758407419000378
},
{
"clip_ratio/high_max": 0.0,
@@ -4356,26 +4356,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 135.0,
- "completions/max_terminated_length": 135.0,
- "completions/mean_length": 68.5,
- "completions/mean_terminated_length": 68.5,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.2759611010551453,
- "epoch": 1.2682926829268293,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12738144397735596,
- "kl": 1.5844128029129934e-05,
- "learning_rate": 5.754996423887061e-07,
- "loss": -0.011055335402488708,
- "num_tokens": 640262.0,
- "reward": 0.344404935836792,
- "reward_std": 0.16542991995811462,
- "rewards/true_env_reward_fn/mean": 0.344404935836792,
- "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
"step": 156,
- "step_time": 5.706334413998775
+ "step_time": 11.009583763999672
},
{
"clip_ratio/high_max": 0.0,
@@ -4384,26 +4384,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.0904476642608643,
- "epoch": 1.2764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.733699021628127e-05,
- "kl": 1.2238857834745431e-05,
- "learning_rate": 5.720905524360308e-07,
- "loss": 6.076299996493617e-07,
- "num_tokens": 645091.0,
- "reward": 0.4731999933719635,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4731999933719635,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
"step": 157,
- "step_time": 3.7696847109982627
+ "step_time": 10.12789283499933
},
{
"clip_ratio/high_max": 0.0,
@@ -4412,26 +4412,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.0,
- "completions/mean_terminated_length": 60.0,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.3856677412986755,
- "epoch": 1.2845528455284554,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18670028448104858,
- "kl": 1.8415606064081658e-05,
- "learning_rate": 5.686660533800736e-07,
- "loss": -0.07078710198402405,
- "num_tokens": 648179.0,
- "reward": 0.537517786026001,
- "reward_std": 0.1451217085123062,
- "rewards/true_env_reward_fn/mean": 0.537517786026001,
- "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
"step": 158,
- "step_time": 3.7075291149994882
+ "step_time": 7.800485663000472
},
{
"clip_ratio/high_max": 0.0,
@@ -4440,26 +4440,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.125,
- "completions/mean_terminated_length": 68.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.121916651725769,
- "epoch": 1.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11415883898735046,
- "kl": 1.909901220642496e-05,
- "learning_rate": 5.652264518527725e-07,
- "loss": -0.04401372745633125,
- "num_tokens": 652044.0,
- "reward": 0.5182899832725525,
- "reward_std": 0.21869486570358276,
- "rewards/true_env_reward_fn/mean": 0.5182899832725525,
- "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
"step": 159,
- "step_time": 3.8929355969994504
+ "step_time": 13.77649076500029
},
{
"clip_ratio/high_max": 0.0,
@@ -4468,26 +4468,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 143.0,
- "completions/max_terminated_length": 143.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5205118060112,
- "epoch": 1.3008130081300813,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.2403375506401062,
- "kl": 3.7574073758150917e-05,
- "learning_rate": 5.617720558383508e-07,
- "loss": 0.26385918259620667,
- "num_tokens": 656362.0,
- "reward": 0.41201668977737427,
- "reward_std": 0.2023741453886032,
- "rewards/true_env_reward_fn/mean": 0.41201668977737427,
- "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
"step": 160,
- "step_time": 6.157555950998358
+ "step_time": 13.25029361300085
},
{
"clip_ratio/high_max": 0.0,
@@ -4496,26 +4496,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3556928038597107,
- "epoch": 1.3089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18135924637317657,
- "kl": 2.6372636057203636e-05,
- "learning_rate": 5.583031746457407e-07,
- "loss": -0.10538280755281448,
- "num_tokens": 659977.0,
- "reward": 0.4239906072616577,
- "reward_std": 0.3287450969219208,
- "rewards/true_env_reward_fn/mean": 0.4239906072616577,
- "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
"step": 161,
- "step_time": 4.709477423999488
+ "step_time": 17.837881629000094
},
{
"clip_ratio/high_max": 0.0,
@@ -4524,26 +4524,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 52.5,
- "completions/mean_terminated_length": 52.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 0.9743769466876984,
- "epoch": 1.3170731707317074,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.178797647356987,
- "kl": 1.2532927030406427e-05,
- "learning_rate": 5.548201188808869e-07,
- "loss": -0.04164513945579529,
- "num_tokens": 661409.0,
- "reward": 0.8041956424713135,
- "reward_std": 0.1363772451877594,
- "rewards/true_env_reward_fn/mean": 0.8041956424713135,
- "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
"step": 162,
- "step_time": 3.1791253910014348
+ "step_time": 9.286757633000889
},
{
"clip_ratio/high_max": 0.0,
@@ -4552,26 +4552,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.3072250485420227,
- "epoch": 1.3252032520325203,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16768279671669006,
- "kl": 1.89386219062726e-05,
- "learning_rate": 5.513232004189339e-07,
- "loss": -0.01292814314365387,
- "num_tokens": 666504.0,
- "reward": 0.27981066703796387,
- "reward_std": 0.2949208915233612,
- "rewards/true_env_reward_fn/mean": 0.27981066703796387,
- "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
"step": 163,
- "step_time": 3.778431355000066
+ "step_time": 10.785562561999996
},
{
"clip_ratio/high_max": 0.0,
@@ -4580,26 +4580,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 51.0,
- "completions/mean_terminated_length": 51.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1984856128692627,
- "epoch": 1.3333333333333333,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13123764097690582,
- "kl": 1.9091786271019373e-05,
- "learning_rate": 5.478127323763027e-07,
- "loss": 0.035523779690265656,
- "num_tokens": 671004.0,
- "reward": 0.2771500051021576,
- "reward_std": 0.30146247148513794,
- "rewards/true_env_reward_fn/mean": 0.2771500051021576,
- "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
"step": 164,
- "step_time": 3.304021460000513
+ "step_time": 12.722446307000155
},
{
"clip_ratio/high_max": 0.0,
@@ -4608,26 +4608,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 102.0,
- "completions/max_terminated_length": 102.0,
- "completions/mean_length": 84.875,
- "completions/mean_terminated_length": 84.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2726752758026123,
- "epoch": 1.3414634146341464,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15594874322414398,
- "kl": 2.5981638827943243e-05,
- "learning_rate": 5.442890290826518e-07,
- "loss": -0.01398652046918869,
- "num_tokens": 677307.0,
- "reward": 0.35573017597198486,
- "reward_std": 0.25944042205810547,
- "rewards/true_env_reward_fn/mean": 0.35573017597198486,
- "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
"step": 165,
- "step_time": 4.972808451999299
+ "step_time": 14.89655208000022
},
{
"clip_ratio/high_max": 0.0,
@@ -4636,26 +4636,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 210.0,
- "completions/max_terminated_length": 210.0,
- "completions/mean_length": 103.625,
- "completions/mean_terminated_length": 103.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0949090719223022,
- "epoch": 1.3495934959349594,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.07191785424947739,
- "kl": 1.4828182884230046e-05,
- "learning_rate": 5.407524060527332e-07,
- "loss": -0.05063021928071976,
- "num_tokens": 682076.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
"step": 166,
- "step_time": 8.769379133000257
+ "step_time": 8.962532588000613
},
{
"clip_ratio/high_max": 0.0,
@@ -4664,26 +4664,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1857684254646301,
- "epoch": 1.3577235772357723,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.923542191041633e-05,
- "kl": 1.0165251751459436e-05,
- "learning_rate": 5.37203179958141e-07,
- "loss": 5.114516170579009e-07,
- "num_tokens": 685500.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
"step": 167,
- "step_time": 3.1906087530005607
+ "step_time": 11.708963983000103
},
{
"clip_ratio/high_max": 0.0,
@@ -4692,26 +4692,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 74.25,
- "completions/mean_terminated_length": 74.25,
- "completions/min_length": 16.0,
- "completions/min_terminated_length": 16.0,
- "entropy": 1.527149498462677,
- "epoch": 1.3658536585365852,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13164250552654266,
- "kl": 1.8541333702160046e-05,
- "learning_rate": 5.33641668598956e-07,
- "loss": -0.2347300797700882,
- "num_tokens": 688318.0,
- "reward": 0.7218117713928223,
- "reward_std": 0.1818692982196808,
- "rewards/true_env_reward_fn/mean": 0.7218117713928223,
- "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
"step": 168,
- "step_time": 8.50137474999974
+ "step_time": 16.905241970000134
},
{
"clip_ratio/high_max": 0.0,
@@ -4720,26 +4720,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 63.0,
- "completions/min_terminated_length": 63.0,
- "entropy": 1.3749513030052185,
- "epoch": 1.3739837398373984,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13246971368789673,
- "kl": 1.2620409506780561e-05,
- "learning_rate": 5.300681908752895e-07,
- "loss": 0.024534843862056732,
- "num_tokens": 692541.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
"step": 169,
- "step_time": 3.9512340759993094
+ "step_time": 12.27045077799994
},
{
"clip_ratio/high_max": 0.0,
@@ -4747,27 +4747,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 67.875,
- "completions/mean_terminated_length": 67.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2032299041748047,
- "epoch": 1.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11185794323682785,
- "kl": 1.5517784049734473e-05,
- "learning_rate": 5.264830667587295e-07,
- "loss": -0.05245225131511688,
- "num_tokens": 698064.0,
- "reward": 0.4444866180419922,
- "reward_std": 0.32400256395339966,
- "rewards/true_env_reward_fn/mean": 0.4444866180419922,
- "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
"step": 170,
- "step_time": 4.471538110999973
+ "step_time": 31.721865252000498
},
{
"clip_ratio/high_max": 0.0,
@@ -4776,26 +4776,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 95.0,
- "completions/mean_terminated_length": 95.0,
- "completions/min_length": 79.0,
- "completions/min_terminated_length": 79.0,
- "entropy": 1.4204387068748474,
- "epoch": 1.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1646534949541092,
- "kl": 2.4697198568901513e-05,
- "learning_rate": 5.228866172636899e-07,
- "loss": 0.02632315456867218,
- "num_tokens": 704196.0,
- "reward": 0.304565966129303,
- "reward_std": 0.32997164130210876,
- "rewards/true_env_reward_fn/mean": 0.304565966129303,
- "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
"step": 171,
- "step_time": 5.0436168590003945
+ "step_time": 8.973740739000277
},
{
"clip_ratio/high_max": 0.0,
@@ -4804,54 +4804,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1254178285598755,
- "epoch": 1.3983739837398375,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15058821439743042,
- "kl": 1.8407325114822015e-05,
- "learning_rate": 5.192791644186662e-07,
- "loss": 0.025478817522525787,
- "num_tokens": 706411.0,
- "reward": 0.7279239892959595,
- "reward_std": 0.11376125365495682,
- "rewards/true_env_reward_fn/mean": 0.7279239892959595,
- "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
"step": 172,
- "step_time": 3.2261944119982218
+ "step_time": 10.39117795600032
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
- "clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0894773602485657,
- "epoch": 1.4065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1687694787979126,
- "kl": 2.6046765015053097e-05,
- "learning_rate": 5.156610312374013e-07,
- "loss": -0.056941211223602295,
- "num_tokens": 711212.0,
- "reward": 0.4907146692276001,
- "reward_std": 0.3376546800136566,
- "rewards/true_env_reward_fn/mean": 0.4907146692276001,
- "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
"step": 173,
- "step_time": 3.9498180619993946
+ "step_time": 11.8111169230001
},
{
"clip_ratio/high_max": 0.0,
@@ -4860,26 +4860,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 60.125,
- "completions/mean_terminated_length": 60.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2238691449165344,
- "epoch": 1.4146341463414633,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19584441184997559,
- "kl": 4.8285241064149886e-05,
- "learning_rate": 5.120325416899629e-07,
- "loss": 0.0766875222325325,
- "num_tokens": 715409.0,
- "reward": 0.4593355059623718,
- "reward_std": 0.3909546136856079,
- "rewards/true_env_reward_fn/mean": 0.4593355059623718,
- "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
"step": 174,
- "step_time": 4.100519798999812
+ "step_time": 17.211099596999702
},
{
"clip_ratio/high_max": 0.0,
@@ -4888,26 +4888,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1763202548027039,
- "epoch": 1.4227642276422765,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1448555737733841,
- "kl": 1.2618989785551094e-05,
- "learning_rate": 5.08394020673734e-07,
- "loss": -0.012558378279209137,
- "num_tokens": 722327.0,
- "reward": 0.15966665744781494,
- "reward_std": 0.3235519230365753,
- "rewards/true_env_reward_fn/mean": 0.15966665744781494,
- "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
"step": 175,
- "step_time": 3.875348296000084
+ "step_time": 12.29185969800028
},
{
"clip_ratio/high_max": 0.0,
@@ -4916,26 +4916,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2852763533592224,
- "epoch": 1.4308943089430894,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22303813695907593,
- "kl": 4.580334098136518e-05,
- "learning_rate": 5.047457939843227e-07,
- "loss": -0.09214464575052261,
- "num_tokens": 726828.0,
- "reward": 0.25830498337745667,
- "reward_std": 0.37860655784606934,
- "rewards/true_env_reward_fn/mean": 0.25830498337745667,
- "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
"step": 176,
- "step_time": 4.241473076999682
+ "step_time": 10.987576109999736
},
{
"clip_ratio/high_max": 0.0,
@@ -4944,26 +4944,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.3609731197357178,
- "epoch": 1.4390243902439024,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.001985745271668e-05,
- "kl": 1.4942165307729738e-05,
- "learning_rate": 5.010881882863893e-07,
- "loss": 7.44550789022469e-07,
- "num_tokens": 729930.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
"step": 177,
- "step_time": 3.3902666960002534
+ "step_time": 10.792315139000493
},
{
"clip_ratio/high_max": 0.0,
@@ -4972,26 +4972,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2678966522216797,
- "epoch": 1.4471544715447155,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15351757407188416,
- "kl": 1.6737100395403104e-05,
- "learning_rate": 4.974215310843967e-07,
- "loss": 0.041131969541311264,
- "num_tokens": 732155.0,
- "reward": 0.7803820371627808,
- "reward_std": 0.08667682856321335,
- "rewards/true_env_reward_fn/mean": 0.7803820371627808,
- "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
"step": 178,
- "step_time": 3.637111981999624
+ "step_time": 12.399353334999887
},
{
"clip_ratio/high_max": 0.0,
@@ -5000,26 +5000,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.146271526813507,
- "epoch": 1.4552845528455285,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18466196954250336,
- "kl": 3.719841197380447e-05,
- "learning_rate": 4.937461506932859e-07,
- "loss": 0.029051154851913452,
- "num_tokens": 735418.0,
- "reward": 0.40377071499824524,
- "reward_std": 0.28345924615859985,
- "rewards/true_env_reward_fn/mean": 0.40377071499824524,
- "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
"step": 179,
- "step_time": 3.340555791999577
+ "step_time": 12.45691162600042
},
{
"clip_ratio/high_max": 0.0,
@@ -5028,26 +5028,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 170.0,
- "completions/max_terminated_length": 170.0,
- "completions/mean_length": 79.75,
- "completions/mean_terminated_length": 79.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.3902945518493652,
- "epoch": 1.4634146341463414,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.1020251139998436,
- "kl": 1.8220500351162627e-05,
- "learning_rate": 4.900623762090777e-07,
- "loss": -0.002344265580177307,
- "num_tokens": 740540.0,
- "reward": 0.3590222895145416,
- "reward_std": 0.12487777322530746,
- "rewards/true_env_reward_fn/mean": 0.3590222895145416,
- "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
"step": 180,
- "step_time": 7.219923718001155
+ "step_time": 12.108760526000424
},
{
"clip_ratio/high_max": 0.0,
@@ -5056,26 +5056,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.517557680606842,
- "epoch": 1.4715447154471546,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24192294478416443,
- "kl": 4.1268089262302965e-05,
- "learning_rate": 4.863705374794055e-07,
- "loss": 0.09132950007915497,
- "num_tokens": 744723.0,
- "reward": 0.23991000652313232,
- "reward_std": 0.2837013602256775,
- "rewards/true_env_reward_fn/mean": 0.23991000652313232,
- "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
"step": 181,
- "step_time": 3.696339096999509
+ "step_time": 14.16400909000049
},
{
"clip_ratio/high_max": 0.0,
@@ -5084,26 +5084,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.249614655971527,
- "epoch": 1.4796747967479675,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.1566745787858963,
- "kl": 2.6629099920683075e-05,
- "learning_rate": 4.826709650739811e-07,
- "loss": 0.003972277045249939,
- "num_tokens": 748979.0,
- "reward": 0.4935140311717987,
- "reward_std": 0.41978561878204346,
- "rewards/true_env_reward_fn/mean": 0.4935140311717987,
- "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
"step": 182,
- "step_time": 3.316512920000605
+ "step_time": 19.767916835999586
},
{
"clip_ratio/high_max": 0.0,
@@ -5112,26 +5112,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1666916608810425,
- "epoch": 1.4878048780487805,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001005820304271765,
- "kl": 1.2246940059412736e-05,
- "learning_rate": 4.789639902549948e-07,
- "loss": 6.330609494398232e-07,
- "num_tokens": 751323.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
"step": 183,
- "step_time": 3.7099916660008603
+ "step_time": 10.428195530999346
},
{
"clip_ratio/high_max": 0.0,
@@ -5140,26 +5140,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.3246222138404846,
- "epoch": 1.4959349593495934,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18140996992588043,
- "kl": 3.1042441150930244e-05,
- "learning_rate": 4.752499449474535e-07,
- "loss": -0.022353097796440125,
- "num_tokens": 755494.0,
- "reward": 0.4561777412891388,
- "reward_std": 0.2439236342906952,
- "rewards/true_env_reward_fn/mean": 0.4561777412891388,
- "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
"step": 184,
- "step_time": 3.7916486710000754
+ "step_time": 16.47171987999991
},
{
"clip_ratio/high_max": 0.0,
@@ -5168,26 +5168,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 75.75,
- "completions/mean_terminated_length": 75.75,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.2625707983970642,
- "epoch": 1.5040650406504064,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11702944338321686,
- "kl": 1.503958355897339e-05,
- "learning_rate": 4.715291617094607e-07,
- "loss": 0.023916304111480713,
- "num_tokens": 758432.0,
- "reward": 0.5015827417373657,
- "reward_std": 0.1783808320760727,
- "rewards/true_env_reward_fn/mean": 0.5015827417373657,
- "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
"step": 185,
- "step_time": 4.226409274000616
+ "step_time": 13.684267182999974
},
{
"clip_ratio/high_max": 0.0,
@@ -5196,26 +5196,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 54.0,
- "completions/mean_terminated_length": 54.0,
- "completions/min_length": 25.0,
- "completions/min_terminated_length": 25.0,
- "entropy": 1.1460023522377014,
- "epoch": 1.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11851620674133301,
- "kl": 1.3728345948038623e-05,
- "learning_rate": 4.678019737024387e-07,
- "loss": 0.0831337422132492,
- "num_tokens": 764336.0,
- "reward": 0.19094166159629822,
- "reward_std": 0.30934420228004456,
- "rewards/true_env_reward_fn/mean": 0.19094166159629822,
- "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
"step": 186,
- "step_time": 3.563357556000483
+ "step_time": 10.281018189999031
},
{
"clip_ratio/high_max": 0.0,
@@ -5224,26 +5224,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1291148662567139,
- "epoch": 1.5203252032520327,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.864256960805506e-05,
- "kl": 1.081683785741916e-05,
- "learning_rate": 4.6406871466129704e-07,
- "loss": 5.412177870312007e-07,
- "num_tokens": 766608.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
"step": 187,
- "step_time": 3.534869859002356
+ "step_time": 24.351223329000277
},
{
"clip_ratio/high_max": 0.0,
@@ -5252,26 +5252,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 92.0,
- "completions/mean_terminated_length": 92.0,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3645328283309937,
- "epoch": 1.5284552845528454,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18168245255947113,
- "kl": 2.6857565899263136e-05,
- "learning_rate": 4.6032971886454956e-07,
- "loss": -0.005156125873327255,
- "num_tokens": 774496.0,
- "reward": 0.11249999701976776,
- "reward_std": 0.20856082439422607,
- "rewards/true_env_reward_fn/mean": 0.11249999701976776,
- "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
"step": 188,
- "step_time": 8.34005261099992
+ "step_time": 10.644911742000204
},
{
"clip_ratio/high_max": 0.0,
@@ -5280,26 +5280,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1495982110500336,
- "epoch": 1.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14486820995807648,
- "kl": 1.3202762147557223e-05,
- "learning_rate": 4.5658532110438337e-07,
- "loss": -0.0010610297322273254,
- "num_tokens": 777186.0,
- "reward": 0.5879127383232117,
- "reward_std": 0.05142820253968239,
- "rewards/true_env_reward_fn/mean": 0.5879127383232117,
- "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
"step": 189,
- "step_time": 3.4856022139993
+ "step_time": 9.579594637999435
},
{
"clip_ratio/high_max": 0.0,
@@ -5308,26 +5308,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.241140365600586,
- "epoch": 1.5447154471544715,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001250067143701017,
- "kl": 1.5482702110602986e-05,
- "learning_rate": 4.52835856656681e-07,
- "loss": 7.80837922320643e-07,
- "num_tokens": 779965.0,
- "reward": 0.6861198544502258,
- "reward_std": 0.08807206153869629,
- "rewards/true_env_reward_fn/mean": 0.6861198544502258,
- "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
"step": 190,
- "step_time": 3.904181735999373
+ "step_time": 11.495368679999501
},
{
"clip_ratio/high_max": 0.0,
@@ -5336,26 +5336,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2944807410240173,
- "epoch": 1.5528455284552845,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2030111849308014,
- "kl": 3.585523518268019e-05,
- "learning_rate": 4.490816612509991e-07,
- "loss": 0.0143373291939497,
- "num_tokens": 786140.0,
- "reward": 0.35173332691192627,
- "reward_std": 0.18115806579589844,
- "rewards/true_env_reward_fn/mean": 0.35173332691192627,
- "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
"step": 191,
- "step_time": 4.005758510000305
+ "step_time": 7.909104242000012
},
{
"clip_ratio/high_max": 0.0,
@@ -5364,26 +5364,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 58.0,
- "completions/mean_terminated_length": 58.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2177271246910095,
- "epoch": 1.5609756097560976,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11188288033008575,
- "kl": 2.3622495064046234e-05,
- "learning_rate": 4.45323071040508e-07,
- "loss": -0.02369789034128189,
- "num_tokens": 790424.0,
- "reward": 0.4888629913330078,
- "reward_std": 0.23310808837413788,
- "rewards/true_env_reward_fn/mean": 0.4888629913330078,
- "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
"step": 192,
- "step_time": 3.2518814809991454
+ "step_time": 10.043768619000275
},
{
"clip_ratio/high_max": 0.0,
@@ -5392,26 +5392,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.192966103553772,
- "epoch": 1.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1378823071718216,
- "kl": 1.7358055174554465e-05,
- "learning_rate": 4.4156042257189143e-07,
- "loss": 0.06256310641765594,
- "num_tokens": 794521.0,
- "reward": 0.5219699740409851,
- "reward_std": 0.06214587390422821,
- "rewards/true_env_reward_fn/mean": 0.5219699740409851,
- "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
"step": 193,
- "step_time": 4.222739491999164
+ "step_time": 10.19648474899941
},
{
"clip_ratio/high_max": 0.0,
@@ -5420,26 +5420,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1770159006118774,
- "epoch": 1.5772357723577235,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19550059735774994,
- "kl": 2.4871268578863237e-05,
- "learning_rate": 4.377940527552125e-07,
- "loss": 0.05841376632452011,
- "num_tokens": 798194.0,
- "reward": 0.42302167415618896,
- "reward_std": 0.2911272346973419,
- "rewards/true_env_reward_fn/mean": 0.42302167415618896,
- "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
"step": 194,
- "step_time": 3.8170270639984665
+ "step_time": 14.372816425000565
},
{
"clip_ratio/high_max": 0.0,
@@ -5448,26 +5448,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.5244255661964417,
- "epoch": 1.5853658536585367,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1872202306985855,
- "kl": 2.5990483663917985e-05,
- "learning_rate": 4.340242988337462e-07,
- "loss": -0.044112429022789,
- "num_tokens": 802802.0,
- "reward": 0.33745431900024414,
- "reward_std": 0.22955451905727386,
- "rewards/true_env_reward_fn/mean": 0.33745431900024414,
- "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
"step": 195,
- "step_time": 4.27381555500142
+ "step_time": 11.282298853000157
},
{
"clip_ratio/high_max": 0.0,
@@ -5476,26 +5476,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.345891296863556,
- "epoch": 1.5934959349593496,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16891054809093475,
- "kl": 2.7261638024356216e-05,
- "learning_rate": 4.3025149835378275e-07,
- "loss": -0.139386385679245,
- "num_tokens": 807881.0,
- "reward": 0.3240283131599426,
- "reward_std": 0.2803676128387451,
- "rewards/true_env_reward_fn/mean": 0.3240283131599426,
- "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
"step": 196,
- "step_time": 4.981287381999209
+ "step_time": 8.65745804199969
},
{
"clip_ratio/high_max": 0.0,
@@ -5504,26 +5504,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.246802031993866,
- "epoch": 1.6016260162601625,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2096729576587677,
- "kl": 3.5958016269432846e-05,
- "learning_rate": 4.2647598913440264e-07,
- "loss": -0.02941281348466873,
- "num_tokens": 812500.0,
- "reward": 0.4126526415348053,
- "reward_std": 0.36393746733665466,
- "rewards/true_env_reward_fn/mean": 0.4126526415348053,
- "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
"step": 197,
- "step_time": 3.101726017999681
+ "step_time": 9.615110594999805
},
{
"clip_ratio/high_max": 0.0,
@@ -5532,26 +5532,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4369062185287476,
- "epoch": 1.6097560975609757,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1862105429172516,
- "kl": 4.7646244638599455e-05,
- "learning_rate": 4.2269810923722965e-07,
- "loss": 0.0521145761013031,
- "num_tokens": 818718.0,
- "reward": 0.26869943737983704,
- "reward_std": 0.1483483463525772,
- "rewards/true_env_reward_fn/mean": 0.26869943737983704,
- "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
"step": 198,
- "step_time": 4.287780451000799
+ "step_time": 10.832162847999825
},
{
"clip_ratio/high_max": 0.0,
@@ -5560,26 +5560,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 21.0,
- "completions/min_terminated_length": 21.0,
- "entropy": 1.192937195301056,
- "epoch": 1.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2013663798570633,
- "kl": 1.8760739294521045e-05,
- "learning_rate": 4.189181969361588e-07,
- "loss": 0.07236722111701965,
- "num_tokens": 825728.0,
- "reward": 0.23110000789165497,
- "reward_std": 0.23212090134620667,
- "rewards/true_env_reward_fn/mean": 0.23110000789165497,
- "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
"step": 199,
- "step_time": 4.708717262998107
+ "step_time": 10.94759418800004
},
{
"clip_ratio/high_max": 0.0,
@@ -5588,26 +5588,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.3792839050292969,
- "epoch": 1.6260162601626016,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.7985117463395e-05,
- "kl": 1.4280476534622721e-05,
- "learning_rate": 4.1513659068706814e-07,
- "loss": 7.153485626076872e-07,
- "num_tokens": 830318.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
"step": 200,
- "step_time": 3.9055351140013954
+ "step_time": 10.322844021000492
},
{
"clip_ratio/high_max": 0.0,
@@ -5616,26 +5616,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.0505937337875366,
- "epoch": 1.6341463414634148,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13188514113426208,
- "kl": 2.842090088961413e-05,
- "learning_rate": 4.1135362909751326e-07,
- "loss": -0.0017508119344711304,
- "num_tokens": 834933.0,
- "reward": 0.36545002460479736,
- "reward_std": 0.24526984989643097,
- "rewards/true_env_reward_fn/mean": 0.36545002460479736,
- "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.083335876464844,
+ "completions/mean_terminated_length": 62.083335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.319727510213852,
+ "epoch": 4.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08774503320455551,
+ "kl": 6.86226876496221e-05,
+ "learning_rate": 6.077002463994907e-07,
+ "loss": 0.058894164860248566,
+ "num_tokens": 4956466.0,
+ "reward": 0.41384777426719666,
+ "reward_std": 0.17193447053432465,
+ "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+ "rewards/true_env_reward_fn/std": 0.17193445563316345,
"step": 201,
- "step_time": 3.895525625997834
+ "step_time": 10.146928047999609
},
{
"clip_ratio/high_max": 0.0,
@@ -5644,26 +5644,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 163.0,
- "completions/max_terminated_length": 163.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.3750707507133484,
- "epoch": 1.6422764227642277,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1798955649137497,
- "kl": 2.5428611479583196e-05,
- "learning_rate": 4.075696508964076e-07,
- "loss": 0.1843666434288025,
- "num_tokens": 838076.0,
- "reward": 0.44641831517219543,
- "reward_std": 0.30040720105171204,
- "rewards/true_env_reward_fn/mean": 0.44641831517219543,
- "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 59.395835876464844,
+ "completions/mean_terminated_length": 59.395835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1935442388057709,
+ "epoch": 4.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08328510820865631,
+ "kl": 7.948942857183283e-05,
+ "learning_rate": 6.035394200708103e-07,
+ "loss": 0.045964501798152924,
+ "num_tokens": 4978733.0,
+ "reward": 0.4896667003631592,
+ "reward_std": 0.3830767571926117,
+ "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+ "rewards/true_env_reward_fn/std": 0.3830767571926117,
"step": 202,
- "step_time": 6.74463491500137
+ "step_time": 10.025533761000133
},
{
"clip_ratio/high_max": 0.0,
@@ -5672,26 +5672,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 79.125,
- "completions/mean_terminated_length": 79.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3369249105453491,
- "epoch": 1.6504065040650406,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 62.22916793823242,
+ "completions/mean_terminated_length": 62.22916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2698631286621094,
+ "epoch": 4.951219512195122,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11249764263629913,
- "kl": 1.1453126717242412e-05,
- "learning_rate": 4.0378499490369267e-07,
- "loss": -0.08210685849189758,
- "num_tokens": 841933.0,
- "reward": 0.3737962245941162,
- "reward_std": 0.13184049725532532,
- "rewards/true_env_reward_fn/mean": 0.3737962245941162,
- "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "grad_norm": 0.07158155739307404,
+ "kl": 8.286665615742095e-05,
+ "learning_rate": 5.993710887521302e-07,
+ "loss": -0.03551984950900078,
+ "num_tokens": 5009440.0,
+ "reward": 0.45639634132385254,
+ "reward_std": 0.1837630569934845,
+ "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+ "rewards/true_env_reward_fn/std": 0.1837630420923233,
"step": 203,
- "step_time": 5.5639925510004105
+ "step_time": 9.823523802999716
},
{
"clip_ratio/high_max": 0.0,
@@ -5700,26 +5700,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.5737199783325195,
- "epoch": 1.6585365853658538,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19394491612911224,
- "kl": 5.05317857459886e-05,
- "learning_rate": 4e-07,
- "loss": 0.05622926354408264,
- "num_tokens": 847711.0,
- "reward": 0.10725000500679016,
- "reward_std": 0.19980257749557495,
- "rewards/true_env_reward_fn/mean": 0.10725000500679016,
- "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "entropy": 1.2825455367565155,
+ "epoch": 4.975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07431793212890625,
+ "kl": 7.247529447340639e-05,
+ "learning_rate": 5.951955545823342e-07,
+ "loss": -0.02418600022792816,
+ "num_tokens": 5036036.0,
+ "reward": 0.4476773142814636,
+ "reward_std": 0.20447416603565216,
+ "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+ "rewards/true_env_reward_fn/std": 0.20447418093681335,
"step": 204,
- "step_time": 4.918089437000162
+ "step_time": 11.960790695000014
},
{
"clip_ratio/high_max": 0.0,
@@ -5728,26 +5728,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 60.5,
- "completions/mean_terminated_length": 60.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.119917094707489,
- "epoch": 1.6666666666666665,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2438389956951141,
+ "epoch": 5.0,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.10553985089063644,
- "kl": 1.2793303994840244e-05,
- "learning_rate": 3.9621500509630725e-07,
- "loss": -0.011355768889188766,
- "num_tokens": 849519.0,
- "reward": 0.6593211889266968,
- "reward_std": 0.11862105131149292,
- "rewards/true_env_reward_fn/mean": 0.6593211889266968,
- "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "grad_norm": 0.06833480298519135,
+ "kl": 5.646793260893901e-05,
+ "learning_rate": 5.91013120222401e-07,
+ "loss": 0.0021926667541265488,
+ "num_tokens": 5063228.0,
+ "reward": 0.4300723075866699,
+ "reward_std": 0.12116922438144684,
+ "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+ "rewards/true_env_reward_fn/std": 0.12116922438144684,
"step": 205,
- "step_time": 4.127652793999005
+ "step_time": 9.874485716000436
},
{
"clip_ratio/high_max": 0.0,
@@ -5756,26 +5756,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 75.125,
- "completions/mean_terminated_length": 75.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3856809735298157,
- "epoch": 1.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1911258101463318,
- "kl": 4.2569914512569085e-05,
- "learning_rate": 3.9243034910359247e-07,
- "loss": 0.10561336576938629,
- "num_tokens": 854156.0,
- "reward": 0.2516202926635742,
- "reward_std": 0.22357939183712006,
- "rewards/true_env_reward_fn/mean": 0.2516202926635742,
- "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "completions/max_length": 205.0,
+ "completions/max_terminated_length": 205.0,
+ "completions/mean_length": 66.1875,
+ "completions/mean_terminated_length": 66.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2645181119441986,
+ "epoch": 5.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08248262107372284,
+ "kl": 9.333990783488844e-05,
+ "learning_rate": 5.868240888334652e-07,
+ "loss": 0.12253005057573318,
+ "num_tokens": 5093373.0,
+ "reward": 0.28542038798332214,
+ "reward_std": 0.326623797416687,
+ "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+ "rewards/true_env_reward_fn/std": 0.326623797416687,
"step": 206,
- "step_time": 4.714620994000143
+ "step_time": 15.706792760000553
},
{
"clip_ratio/high_max": 0.0,
@@ -5784,26 +5784,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.4012945890426636,
- "epoch": 1.6829268292682928,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226125180721283,
- "kl": 5.4918069963605376e-05,
- "learning_rate": 3.886463709024868e-07,
- "loss": 0.012949362397193909,
- "num_tokens": 856151.0,
- "reward": 0.6204804182052612,
- "reward_std": 0.44673967361450195,
- "rewards/true_env_reward_fn/mean": 0.6204804182052612,
- "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.16667175292969,
+ "completions/mean_terminated_length": 67.16667175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.34083291888237,
+ "epoch": 5.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07931552827358246,
+ "kl": 0.00010715152166085318,
+ "learning_rate": 5.826287640548424e-07,
+ "loss": 0.0005592256784439087,
+ "num_tokens": 5118933.0,
+ "reward": 0.5035215616226196,
+ "reward_std": 0.304157018661499,
+ "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+ "rewards/true_env_reward_fn/std": 0.30415698885917664,
"step": 207,
- "step_time": 4.094810713999323
+ "step_time": 10.04168460900064
},
{
"clip_ratio/high_max": 0.0,
@@ -5812,26 +5812,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3708943128585815,
- "epoch": 1.6910569105691056,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12903675436973572,
- "kl": 1.4145812656352064e-05,
- "learning_rate": 3.8486340931293187e-07,
- "loss": -0.07838249206542969,
- "num_tokens": 859380.0,
- "reward": 0.6400156021118164,
- "reward_std": 0.1022576317191124,
- "rewards/true_env_reward_fn/mean": 0.6400156021118164,
- "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 73.4375,
+ "completions/mean_terminated_length": 73.4375,
+ "completions/min_length": 11.0,
+ "completions/min_terminated_length": 11.0,
+ "entropy": 1.1821868121623993,
+ "epoch": 5.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07619761675596237,
+ "kl": 0.00013189401397539768,
+ "learning_rate": 5.784274499820213e-07,
+ "loss": -0.011967688798904419,
+ "num_tokens": 5148682.0,
+ "reward": 0.36097532510757446,
+ "reward_std": 0.29208436608314514,
+ "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+ "rewards/true_env_reward_fn/std": 0.29208436608314514,
"step": 208,
- "step_time": 4.297900428997309
+ "step_time": 15.226898961000188
},
{
"clip_ratio/high_max": 0.0,
@@ -5840,26 +5840,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.75,
- "completions/mean_terminated_length": 56.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2782961130142212,
- "epoch": 1.6991869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12437833100557327,
- "kl": 1.4473939245362999e-05,
- "learning_rate": 3.8108180306384135e-07,
- "loss": -0.036324724555015564,
- "num_tokens": 862714.0,
- "reward": 0.43419933319091797,
- "reward_std": 0.15345513820648193,
- "rewards/true_env_reward_fn/mean": 0.43419933319091797,
- "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 75.39583587646484,
+ "completions/mean_terminated_length": 75.39583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2636725008487701,
+ "epoch": 5.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06586338579654694,
+ "kl": 1.9426477138040354e-05,
+ "learning_rate": 5.742204511446203e-07,
+ "loss": -0.047095417976379395,
+ "num_tokens": 5173205.0,
+ "reward": 0.5261925458908081,
+ "reward_std": 0.2689943313598633,
+ "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+ "rewards/true_env_reward_fn/std": 0.2689943313598633,
"step": 209,
- "step_time": 3.47861851900052
+ "step_time": 23.690397457000017
},
{
"clip_ratio/high_max": 0.0,
@@ -5867,27 +5867,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 123.875,
- "completions/mean_terminated_length": 68.42857360839844,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.5170292258262634,
- "epoch": 1.7073170731707317,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 54.3125,
+ "completions/mean_terminated_length": 54.3125,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2415392696857452,
+ "epoch": 5.121951219512195,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1543467938899994,
- "kl": 2.3121557205740828e-05,
- "learning_rate": 3.7730189076277037e-07,
- "loss": -0.3810324966907501,
- "num_tokens": 869789.0,
- "reward": 0.3795333504676819,
- "reward_std": 0.13815106451511383,
- "rewards/true_env_reward_fn/mean": 0.3795333504676819,
- "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "grad_norm": 0.0990133062005043,
+ "kl": 7.119746442185715e-05,
+ "learning_rate": 5.700080724843146e-07,
+ "loss": -0.030588299036026,
+ "num_tokens": 5199892.0,
+ "reward": 0.3418487012386322,
+ "reward_std": 0.36353805661201477,
+ "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+ "rewards/true_env_reward_fn/std": 0.3635380268096924,
"step": 210,
- "step_time": 20.698896928999602
+ "step_time": 9.864614251999683
},
{
"clip_ratio/high_max": 0.0,
@@ -5896,26 +5896,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2324861884117126,
- "epoch": 1.7154471544715446,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11762262135744095,
- "kl": 1.9743249595194357e-05,
- "learning_rate": 3.735240108655973e-07,
- "loss": -0.09104303270578384,
- "num_tokens": 872604.0,
- "reward": 0.5916227102279663,
- "reward_std": 0.1752101182937622,
- "rewards/true_env_reward_fn/mean": 0.5916227102279663,
- "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2508135735988617,
+ "epoch": 5.146341463414634,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10835102945566177,
+ "kl": 0.00012395972225931473,
+ "learning_rate": 5.657906193327324e-07,
+ "loss": 0.08621911704540253,
+ "num_tokens": 5220141.0,
+ "reward": 0.4987506866455078,
+ "reward_std": 0.27843451499938965,
+ "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+ "rewards/true_env_reward_fn/std": 0.27843451499938965,
"step": 211,
- "step_time": 5.218213289999767
+ "step_time": 9.983622502000344
},
{
"clip_ratio/high_max": 0.0,
@@ -5924,26 +5924,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.4448966979980469,
- "epoch": 1.7235772357723578,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1924143135547638,
- "kl": 3.186432604707079e-05,
- "learning_rate": 3.697485016462174e-07,
- "loss": 0.025449808686971664,
- "num_tokens": 877921.0,
- "reward": 0.1028124988079071,
- "reward_std": 0.2254277467727661,
- "rewards/true_env_reward_fn/mean": 0.1028124988079071,
- "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1611086130142212,
+ "epoch": 5.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08423946797847748,
+ "kl": 4.4347343191475375e-05,
+ "learning_rate": 5.615683973893234e-07,
+ "loss": 0.08670675754547119,
+ "num_tokens": 5244470.0,
+ "reward": 0.4252437949180603,
+ "reward_std": 0.2996494472026825,
+ "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+ "rewards/true_env_reward_fn/std": 0.2996494472026825,
"step": 212,
- "step_time": 3.415528882000217
+ "step_time": 18.68646409699977
},
{
"clip_ratio/high_max": 0.0,
@@ -5952,26 +5952,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
"completions/min_length": 32.0,
"completions/min_terminated_length": 32.0,
- "entropy": 1.185910701751709,
- "epoch": 1.7317073170731707,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001214127623825334,
- "kl": 1.3199866316426778e-05,
- "learning_rate": 3.659757011662538e-07,
- "loss": 6.776077725589857e-07,
- "num_tokens": 880344.0,
- "reward": 0.7329437732696533,
- "reward_std": 0.22123214602470398,
- "rewards/true_env_reward_fn/mean": 0.7329437732696533,
- "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "entropy": 1.3110275566577911,
+ "epoch": 5.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06817487627267838,
+ "kl": 0.00010393545744591393,
+ "learning_rate": 5.573417126992002e-07,
+ "loss": 0.04062439873814583,
+ "num_tokens": 5271369.0,
+ "reward": 0.4340522885322571,
+ "reward_std": 0.26940545439720154,
+ "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+ "rewards/true_env_reward_fn/std": 0.26940542459487915,
"step": 213,
- "step_time": 3.5156538789997285
+ "step_time": 11.3590317649996
},
{
"clip_ratio/high_max": 0.0,
@@ -5980,26 +5980,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 162.0,
- "completions/max_terminated_length": 162.0,
- "completions/mean_length": 90.75,
- "completions/mean_terminated_length": 90.75,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2697569727897644,
- "epoch": 1.7398373983739837,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 54.4375,
+ "completions/mean_terminated_length": 54.4375,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2018881738185883,
+ "epoch": 5.219512195121951,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1553589552640915,
- "kl": 1.621047795197228e-05,
- "learning_rate": 3.622059472447875e-07,
- "loss": -0.111361563205719,
- "num_tokens": 885006.0,
- "reward": 0.5490846633911133,
- "reward_std": 0.14710450172424316,
- "rewards/true_env_reward_fn/mean": 0.5490846633911133,
- "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "grad_norm": 0.10059589147567749,
+ "kl": 0.00011373830420779996,
+ "learning_rate": 5.531108716309547e-07,
+ "loss": -0.09816272556781769,
+ "num_tokens": 5288638.0,
+ "reward": 0.5691710710525513,
+ "reward_std": 0.25253745913505554,
+ "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+ "rewards/true_env_reward_fn/std": 0.25253745913505554,
"step": 214,
- "step_time": 7.051423932000034
+ "step_time": 9.633293675000004
},
{
"clip_ratio/high_max": 0.0,
@@ -6008,26 +6008,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1904898285865784,
- "epoch": 1.7479674796747968,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19009968638420105,
- "kl": 1.6167180092452327e-05,
- "learning_rate": 3.5843957742810864e-07,
- "loss": -0.006048411130905151,
- "num_tokens": 888255.0,
- "reward": 0.5893601179122925,
- "reward_std": 0.16021940112113953,
- "rewards/true_env_reward_fn/mean": 0.5893601179122925,
- "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1958912312984467,
+ "epoch": 5.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0779944583773613,
+ "kl": 7.025236300250981e-05,
+ "learning_rate": 5.488761808544509e-07,
+ "loss": 0.03371567651629448,
+ "num_tokens": 5312081.0,
+ "reward": 0.44519662857055664,
+ "reward_std": 0.22201679646968842,
+ "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+ "rewards/true_env_reward_fn/std": 0.22201678156852722,
"step": 215,
- "step_time": 3.0270869319992926
+ "step_time": 12.463993090999338
},
{
"clip_ratio/high_max": 0.0,
@@ -6036,26 +6036,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2750649452209473,
- "epoch": 1.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1521027386188507,
- "kl": 4.317680577514693e-05,
- "learning_rate": 3.5467692895949205e-07,
- "loss": -0.04247616231441498,
- "num_tokens": 892490.0,
- "reward": 0.1841849982738495,
- "reward_std": 0.30015870928764343,
- "rewards/true_env_reward_fn/mean": 0.1841849982738495,
- "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.2146256864070892,
+ "epoch": 5.2682926829268295,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06834157556295395,
+ "kl": 0.00010125362859980669,
+ "learning_rate": 5.446379473185971e-07,
+ "loss": -0.02198379673063755,
+ "num_tokens": 5334238.0,
+ "reward": 0.5273472666740417,
+ "reward_std": 0.21954773366451263,
+ "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+ "rewards/true_env_reward_fn/std": 0.21954771876335144,
"step": 216,
- "step_time": 3.461749838998003
+ "step_time": 15.126136884000061
},
{
"clip_ratio/high_max": 0.0,
@@ -6064,26 +6064,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2730335593223572,
- "epoch": 1.7642276422764227,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11831019073724747,
- "kl": 2.1095927877468057e-05,
- "learning_rate": 3.509183387490009e-07,
- "loss": -0.051231447607278824,
- "num_tokens": 894688.0,
- "reward": 0.49520131945610046,
- "reward_std": 0.3804744482040405,
- "rewards/true_env_reward_fn/mean": 0.49520131945610046,
- "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 61.333335876464844,
+ "completions/mean_terminated_length": 61.333335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2872387170791626,
+ "epoch": 5.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09578828513622284,
+ "kl": 7.404103780572768e-05,
+ "learning_rate": 5.403964782290961e-07,
+ "loss": -0.041091397404670715,
+ "num_tokens": 5351798.0,
+ "reward": 0.5464547872543335,
+ "reward_std": 0.16224616765975952,
+ "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+ "rewards/true_env_reward_fn/std": 0.16224615275859833,
"step": 217,
- "step_time": 3.7184635590019752
+ "step_time": 7.2901647220001
},
{
"clip_ratio/high_max": 0.0,
@@ -6092,26 +6092,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 47.75,
- "completions/mean_terminated_length": 47.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3015827536582947,
- "epoch": 1.7723577235772359,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21724702417850494,
- "kl": 3.562447545846226e-05,
- "learning_rate": 3.471641433433191e-07,
- "loss": -0.028775859624147415,
- "num_tokens": 899066.0,
- "reward": 0.3376166820526123,
- "reward_std": 0.21976198256015778,
- "rewards/true_env_reward_fn/mean": 0.3376166820526123,
- "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 84.89583587646484,
+ "completions/mean_terminated_length": 84.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.4322342276573181,
+ "epoch": 5.317073170731708,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0667513981461525,
+ "kl": 2.497344303264981e-05,
+ "learning_rate": 5.361520810261778e-07,
+ "loss": -0.06392769515514374,
+ "num_tokens": 5375369.0,
+ "reward": 0.5213420391082764,
+ "reward_std": 0.1436246931552887,
+ "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+ "rewards/true_env_reward_fn/std": 0.1436246782541275,
"step": 218,
- "step_time": 3.344433074000335
+ "step_time": 12.217936152999755
},
{
"clip_ratio/high_max": 0.0,
@@ -6120,26 +6120,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1390373706817627,
- "epoch": 1.7804878048780488,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001077545020962134,
- "kl": 1.554161144667887e-05,
- "learning_rate": 3.434146788956166e-07,
- "loss": 7.676237032683275e-07,
- "num_tokens": 902263.0,
- "reward": 0.5682899951934814,
- "reward_std": 0.1217179074883461,
- "rewards/true_env_reward_fn/mean": 0.5682899951934814,
- "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2382279634475708,
+ "epoch": 5.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07372384518384933,
+ "kl": 9.740726909512887e-05,
+ "learning_rate": 5.319050633623141e-07,
+ "loss": 0.010912742465734482,
+ "num_tokens": 5397956.0,
+ "reward": 0.5623860359191895,
+ "reward_std": 0.285375714302063,
+ "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+ "rewards/true_env_reward_fn/std": 0.285375714302063,
"step": 219,
- "step_time": 3.0424018219982827
+ "step_time": 13.449634822999997
},
{
"clip_ratio/high_max": 0.0,
@@ -6148,26 +6148,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1991714239120483,
- "epoch": 1.7886178861788617,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11428863555192947,
- "kl": 1.1265870853094384e-05,
- "learning_rate": 3.3967028113545045e-07,
- "loss": -0.02927359938621521,
- "num_tokens": 906206.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 65.39583587646484,
+ "completions/mean_terminated_length": 65.39583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2669419348239899,
+ "epoch": 5.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07886125147342682,
+ "kl": 0.00011165817522851285,
+ "learning_rate": 5.276557330799203e-07,
+ "loss": -0.02433733269572258,
+ "num_tokens": 5418511.0,
+ "reward": 0.6075118780136108,
+ "reward_std": 0.30748677253723145,
+ "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+ "rewards/true_env_reward_fn/std": 0.30748677253723145,
"step": 220,
- "step_time": 3.5531271640011255
+ "step_time": 12.051496982000117
},
{
"clip_ratio/high_max": 0.0,
@@ -6176,26 +6176,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2546668648719788,
- "epoch": 1.796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1504630595445633,
- "kl": 1.5663241811125772e-05,
- "learning_rate": 3.3593128533870314e-07,
- "loss": 0.04806854575872421,
- "num_tokens": 909149.0,
- "reward": 0.49599751830101013,
- "reward_std": 0.15429075062274933,
- "rewards/true_env_reward_fn/mean": 0.49599751830101013,
- "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2560602128505707,
+ "epoch": 5.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08607357740402222,
+ "kl": 8.424731549894204e-05,
+ "learning_rate": 5.234043981890393e-07,
+ "loss": 0.013634156435728073,
+ "num_tokens": 5439690.0,
+ "reward": 0.5080039501190186,
+ "reward_std": 0.21975299715995789,
+ "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+ "rewards/true_env_reward_fn/std": 0.21975299715995789,
"step": 221,
- "step_time": 3.1345955030010373
+ "step_time": 10.435893627000496
},
{
"clip_ratio/high_max": 0.0,
@@ -6204,26 +6204,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1428714394569397,
- "epoch": 1.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.310600969707593e-05,
- "kl": 1.1706195891747484e-05,
- "learning_rate": 3.321980262975613e-07,
- "loss": 5.957842290627013e-07,
- "num_tokens": 914211.0,
- "reward": 0.3258306384086609,
- "reward_std": 0.4338511824607849,
- "rewards/true_env_reward_fn/mean": 0.3258306384086609,
- "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3081265985965729,
+ "epoch": 5.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0876006931066513,
+ "kl": 9.127605972025776e-05,
+ "learning_rate": 5.191513668450177e-07,
+ "loss": -0.01968565583229065,
+ "num_tokens": 5468216.0,
+ "reward": 0.3679848611354828,
+ "reward_std": 0.237859845161438,
+ "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+ "rewards/true_env_reward_fn/std": 0.237859845161438,
"step": 222,
- "step_time": 3.8445859539988305
+ "step_time": 11.524256381999294
},
{
"clip_ratio/high_max": 0.0,
@@ -6232,26 +6232,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1629019975662231,
- "epoch": 1.8130081300813008,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010947151895379648,
- "kl": 1.3530024261854123e-05,
- "learning_rate": 3.2847083829053923e-07,
- "loss": 6.723923888785066e-07,
- "num_tokens": 916850.0,
- "reward": 0.5956059694290161,
- "reward_std": 0.09251586347818375,
- "rewards/true_env_reward_fn/mean": 0.5956059694290161,
- "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 67.66667175292969,
+ "completions/mean_terminated_length": 67.66667175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2990808486938477,
+ "epoch": 5.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07685395330190659,
+ "kl": 0.00014209141045284923,
+ "learning_rate": 5.148969473261679e-07,
+ "loss": -0.01107364147901535,
+ "num_tokens": 5488040.0,
+ "reward": 0.5435788035392761,
+ "reward_std": 0.35419100522994995,
+ "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+ "rewards/true_env_reward_fn/std": 0.35419100522994995,
"step": 223,
- "step_time": 3.3021794950000185
+ "step_time": 11.945272217000365
},
{
"clip_ratio/high_max": 0.0,
@@ -6260,26 +6260,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1237311363220215,
- "epoch": 1.821138211382114,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.284348964691162,
+ "epoch": 5.463414634146342,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14958374202251434,
- "kl": 4.083753810846247e-05,
- "learning_rate": 3.2475005505254657e-07,
- "loss": 0.09789139032363892,
- "num_tokens": 919901.0,
- "reward": 0.4179220199584961,
- "reward_std": 0.2486819326877594,
- "rewards/true_env_reward_fn/mean": 0.4179220199584961,
- "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "grad_norm": 0.058678388595581055,
+ "kl": 3.3754420201148605e-05,
+ "learning_rate": 5.106414480114238e-07,
+ "loss": -0.01714605651795864,
+ "num_tokens": 5513005.0,
+ "reward": 0.43836766481399536,
+ "reward_std": 0.26744046807289124,
+ "rewards/true_env_reward_fn/mean": 0.438367635011673,
+ "rewards/true_env_reward_fn/std": 0.26744046807289124,
"step": 224,
- "step_time": 4.954825423001239
+ "step_time": 10.563708176000091
},
{
"clip_ratio/high_max": 0.0,
@@ -6288,26 +6288,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 52.375,
- "completions/mean_terminated_length": 52.375,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1394256949424744,
- "epoch": 1.8292682926829267,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1480269879102707,
- "kl": 3.2915593692450784e-05,
- "learning_rate": 3.210360097450052e-07,
- "loss": 0.11319300532341003,
- "num_tokens": 923840.0,
- "reward": 0.4335233271121979,
- "reward_std": 0.31114333868026733,
- "rewards/true_env_reward_fn/mean": 0.4335233271121979,
- "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3357127904891968,
+ "epoch": 5.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07269197702407837,
+ "kl": 3.727909597728285e-05,
+ "learning_rate": 5.063851773579869e-07,
+ "loss": -0.01076439768075943,
+ "num_tokens": 5530655.0,
+ "reward": 0.6185358762741089,
+ "reward_std": 0.19721543788909912,
+ "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+ "rewards/true_env_reward_fn/std": 0.19721543788909912,
"step": 225,
- "step_time": 3.777259659000265
+ "step_time": 10.372150705000422
},
{
"clip_ratio/high_max": 0.0,
@@ -6316,26 +6316,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.354669451713562,
- "epoch": 1.8373983739837398,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13672730326652527,
- "kl": 2.2323275516100693e-05,
- "learning_rate": 3.173290349260188e-07,
- "loss": -0.0521523654460907,
- "num_tokens": 927882.0,
- "reward": 0.57341468334198,
- "reward_std": 0.11576741933822632,
- "rewards/true_env_reward_fn/mean": 0.57341468334198,
- "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 64.35417175292969,
+ "completions/mean_terminated_length": 64.35417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2975924611091614,
+ "epoch": 5.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0875314474105835,
+ "kl": 9.186910938296933e-05,
+ "learning_rate": 5.021284438789693e-07,
+ "loss": 0.048179637640714645,
+ "num_tokens": 5554032.0,
+ "reward": 0.45552024245262146,
+ "reward_std": 0.31553781032562256,
+ "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+ "rewards/true_env_reward_fn/std": 0.31553778052330017,
"step": 226,
- "step_time": 3.613498073998926
+ "step_time": 12.18860100899974
},
{
"clip_ratio/high_max": 0.0,
@@ -6344,26 +6344,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1750767230987549,
- "epoch": 1.845528455284553,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.230674147605896,
- "kl": 1.800864629331045e-05,
- "learning_rate": 3.136294625205945e-07,
- "loss": 0.058730173856019974,
- "num_tokens": 930225.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 63.208335876464844,
+ "completions/mean_terminated_length": 63.208335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2268341779708862,
+ "epoch": 5.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08318481594324112,
+ "kl": 8.052505017985823e-05,
+ "learning_rate": 4.978715561210307e-07,
+ "loss": 0.027963606640696526,
+ "num_tokens": 5582898.0,
+ "reward": 0.36310288310050964,
+ "reward_std": 0.3131144344806671,
+ "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+ "rewards/true_env_reward_fn/std": 0.3131144344806671,
"step": 227,
- "step_time": 3.3819083769976714
+ "step_time": 12.510411257000214
},
{
"clip_ratio/high_max": 0.0,
@@ -6372,26 +6372,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 40.875,
- "completions/mean_terminated_length": 40.875,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.2825847864151,
- "epoch": 1.8536585365853657,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.30391862988471985,
- "kl": 2.0106223928451072e-05,
- "learning_rate": 3.0993762379092235e-07,
- "loss": -0.10232458263635635,
- "num_tokens": 933048.0,
- "reward": 0.45667415857315063,
- "reward_std": 0.26290765404701233,
- "rewards/true_env_reward_fn/mean": 0.45667415857315063,
- "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 64.9375,
+ "completions/mean_terminated_length": 64.9375,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1668500006198883,
+ "epoch": 5.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0727877989411354,
+ "kl": 7.824771091691218e-05,
+ "learning_rate": 4.936148226420132e-07,
+ "loss": -0.01629078947007656,
+ "num_tokens": 5604791.0,
+ "reward": 0.5653349161148071,
+ "reward_std": 0.2849632799625397,
+ "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+ "rewards/true_env_reward_fn/std": 0.2849632799625397,
"step": 228,
- "step_time": 2.698590726000475
+ "step_time": 10.083805716999905
},
{
"clip_ratio/high_max": 0.0,
@@ -6400,26 +6400,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 41.125,
- "completions/mean_terminated_length": 41.125,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.3104369640350342,
- "epoch": 1.8617886178861789,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.28224605321884155,
- "kl": 6.227439189387951e-05,
- "learning_rate": 3.06253849306714e-07,
- "loss": -0.008679002523422241,
- "num_tokens": 938217.0,
- "reward": 0.2567799985408783,
- "reward_std": 0.2555168867111206,
- "rewards/true_env_reward_fn/mean": 0.2567799985408783,
- "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 68.29167175292969,
+ "completions/mean_terminated_length": 68.29167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.307439923286438,
+ "epoch": 5.585365853658536,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06797961890697479,
+ "kl": 6.601300174224889e-05,
+ "learning_rate": 4.893585519885763e-07,
+ "loss": -0.036383360624313354,
+ "num_tokens": 5631629.0,
+ "reward": 0.49251794815063477,
+ "reward_std": 0.22127023339271545,
+ "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+ "rewards/true_env_reward_fn/std": 0.22127023339271545,
"step": 229,
- "step_time": 2.7451177700022527
+ "step_time": 13.650024606999523
},
{
"clip_ratio/high_max": 0.0,
@@ -6428,26 +6428,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0164751410484314,
- "epoch": 1.8699186991869918,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011273886048002169,
- "kl": 1.1985231139988173e-05,
- "learning_rate": 3.0257846891560323e-07,
- "loss": 5.987301960885816e-07,
- "num_tokens": 941049.0,
- "reward": 0.6971603631973267,
- "reward_std": 0.2594861686229706,
- "rewards/true_env_reward_fn/mean": 0.6971603631973267,
- "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3524385392665863,
+ "epoch": 5.609756097560975,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.12484809756278992,
+ "kl": 0.0002558713749749586,
+ "learning_rate": 4.85103052673832e-07,
+ "loss": -0.0045075975358486176,
+ "num_tokens": 5666712.0,
+ "reward": 0.2605232000350952,
+ "reward_std": 0.22926004230976105,
+ "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+ "rewards/true_env_reward_fn/std": 0.22926005721092224,
"step": 230,
- "step_time": 2.917641182999432
+ "step_time": 15.322059910999542
},
{
"clip_ratio/high_max": 0.0,
@@ -6456,26 +6456,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 13.0,
- "completions/min_terminated_length": 13.0,
- "entropy": 1.1991845965385437,
- "epoch": 1.8780487804878048,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015363915008492768,
- "kl": 1.4284144981502322e-05,
- "learning_rate": 2.989118117136107e-07,
- "loss": 6.798551339670666e-07,
- "num_tokens": 945403.0,
- "reward": 0.31745320558547974,
- "reward_std": 0.42489534616470337,
- "rewards/true_env_reward_fn/mean": 0.31745320558547974,
- "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2202666401863098,
+ "epoch": 5.634146341463414,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07976125925779343,
+ "kl": 8.684267686476232e-05,
+ "learning_rate": 4.808486331549823e-07,
+ "loss": 0.012383833527565002,
+ "num_tokens": 5702004.0,
+ "reward": 0.20571085810661316,
+ "reward_std": 0.2265808880329132,
+ "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+ "rewards/true_env_reward_fn/std": 0.2265808880329132,
"step": 231,
- "step_time": 5.5585464220002905
+ "step_time": 14.180213787999492
},
{
"clip_ratio/high_max": 0.0,
@@ -6484,26 +6484,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2020843029022217,
- "epoch": 1.886178861788618,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13684362173080444,
- "kl": 3.651866154541494e-05,
- "learning_rate": 2.952542060156773e-07,
- "loss": 0.025869816541671753,
- "num_tokens": 949126.0,
- "reward": 0.3984018564224243,
- "reward_std": 0.4202974736690521,
- "rewards/true_env_reward_fn/mean": 0.3984018564224243,
- "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "completions/max_length": 312.0,
+ "completions/max_terminated_length": 312.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3096380531787872,
+ "epoch": 5.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08384906500577927,
+ "kl": 8.733692084206268e-05,
+ "learning_rate": 4.7659560181096067e-07,
+ "loss": 0.02098608762025833,
+ "num_tokens": 5729780.0,
+ "reward": 0.4599723219871521,
+ "reward_std": 0.21680118143558502,
+ "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+ "rewards/true_env_reward_fn/std": 0.21680118143558502,
"step": 232,
- "step_time": 3.4383463869999105
+ "step_time": 21.68401394800003
},
{
"clip_ratio/high_max": 0.0,
@@ -6512,26 +6512,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 58.625,
- "completions/mean_terminated_length": 58.625,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2224581837654114,
- "epoch": 1.8943089430894309,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20249564945697784,
- "kl": 3.106597978330683e-05,
- "learning_rate": 2.9160597932626605e-07,
- "loss": -0.003915777429938316,
- "num_tokens": 952815.0,
- "reward": 0.4141089916229248,
- "reward_std": 0.3064958453178406,
- "rewards/true_env_reward_fn/mean": 0.4141089916229248,
- "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 53.520835876464844,
+ "completions/mean_terminated_length": 53.520835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2158988416194916,
+ "epoch": 5.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08225176483392715,
+ "kl": 9.274652438762132e-05,
+ "learning_rate": 4.7234426692007977e-07,
+ "loss": -0.034079790115356445,
+ "num_tokens": 5753397.0,
+ "reward": 0.5273070335388184,
+ "reward_std": 0.2194610834121704,
+ "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+ "rewards/true_env_reward_fn/std": 0.21946106851100922,
"step": 233,
- "step_time": 3.6410487339999236
+ "step_time": 7.915307780000603
},
{
"clip_ratio/high_max": 0.0,
@@ -6540,26 +6540,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1212781071662903,
- "epoch": 1.9024390243902438,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 78.0,
+ "completions/mean_terminated_length": 78.0,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2362721860408783,
+ "epoch": 5.7073170731707314,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14502985775470734,
- "kl": 3.0683338081871625e-05,
- "learning_rate": 2.879674583100372e-07,
- "loss": -0.0860406681895256,
- "num_tokens": 958656.0,
- "reward": 0.30326664447784424,
- "reward_std": 0.24800051748752594,
- "rewards/true_env_reward_fn/mean": 0.30326664447784424,
- "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "grad_norm": 0.06290840357542038,
+ "kl": 7.484563047910342e-05,
+ "learning_rate": 4.6809493663768575e-07,
+ "loss": -0.008873865008354187,
+ "num_tokens": 5778869.0,
+ "reward": 0.5283026695251465,
+ "reward_std": 0.195759579539299,
+ "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+ "rewards/true_env_reward_fn/std": 0.195759579539299,
"step": 234,
- "step_time": 3.8798253620007017
+ "step_time": 10.278297286999987
},
{
"clip_ratio/high_max": 0.0,
@@ -6568,26 +6568,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 60.75,
- "completions/mean_terminated_length": 60.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.425286054611206,
- "epoch": 1.910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2035023272037506,
- "kl": 5.512987627298571e-05,
- "learning_rate": 2.843389687625986e-07,
- "loss": -0.010439477860927582,
- "num_tokens": 962638.0,
- "reward": 0.3157375156879425,
- "reward_std": 0.46047845482826233,
- "rewards/true_env_reward_fn/mean": 0.3157375156879425,
- "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 64.47917175292969,
+ "completions/mean_terminated_length": 64.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3364675641059875,
+ "epoch": 5.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09685558825731277,
+ "kl": 0.00013002969353692606,
+ "learning_rate": 4.638479189738224e-07,
+ "loss": 0.05070740357041359,
+ "num_tokens": 5804436.0,
+ "reward": 0.44921523332595825,
+ "reward_std": 0.22693434357643127,
+ "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+ "rewards/true_env_reward_fn/std": 0.22693434357643127,
"step": 235,
- "step_time": 4.114513064998391
+ "step_time": 10.754199091999908
},
{
"clip_ratio/high_max": 0.0,
@@ -6596,26 +6596,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 120.0,
- "completions/max_terminated_length": 120.0,
- "completions/mean_length": 77.625,
- "completions/mean_terminated_length": 77.625,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4385854601860046,
- "epoch": 1.91869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12982772290706635,
- "kl": 1.2591926861205138e-05,
- "learning_rate": 2.807208355813339e-07,
- "loss": 0.09093751758337021,
- "num_tokens": 965755.0,
- "reward": 0.6140732765197754,
- "reward_std": 0.27462607622146606,
- "rewards/true_env_reward_fn/mean": 0.6140732765197754,
- "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2267529368400574,
+ "epoch": 5.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10451620817184448,
+ "kl": 6.754635069228243e-05,
+ "learning_rate": 4.596035217709039e-07,
+ "loss": -0.02925686165690422,
+ "num_tokens": 5826895.0,
+ "reward": 0.49523353576660156,
+ "reward_std": 0.15093794465065002,
+ "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+ "rewards/true_env_reward_fn/std": 0.15093792974948883,
"step": 236,
- "step_time": 5.147667763001664
+ "step_time": 9.666070583999954
},
{
"clip_ratio/high_max": 0.0,
@@ -6624,26 +6624,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 71.375,
- "completions/mean_terminated_length": 71.375,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.3476852178573608,
- "epoch": 1.9268292682926829,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14480431377887726,
- "kl": 3.8014684832887724e-05,
- "learning_rate": 2.771133827363101e-07,
- "loss": -0.07322391867637634,
- "num_tokens": 970918.0,
- "reward": 0.3427826166152954,
- "reward_std": 0.42430612444877625,
- "rewards/true_env_reward_fn/mean": 0.3427826166152954,
- "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 67.79167175292969,
+ "completions/mean_terminated_length": 67.79167175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2110519707202911,
+ "epoch": 5.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07890205830335617,
+ "kl": 0.0001245876064785989,
+ "learning_rate": 4.5536205268140286e-07,
+ "loss": -0.03693925589323044,
+ "num_tokens": 5859973.0,
+ "reward": 0.2979053258895874,
+ "reward_std": 0.23015481233596802,
+ "rewards/true_env_reward_fn/mean": 0.297905296087265,
+ "rewards/true_env_reward_fn/std": 0.2301548272371292,
"step": 237,
- "step_time": 4.67846887900123
+ "step_time": 13.971699990999241
},
{
"clip_ratio/high_max": 0.0,
@@ -6652,26 +6652,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 221.0,
- "completions/max_terminated_length": 221.0,
- "completions/mean_length": 94.25,
- "completions/mean_terminated_length": 94.25,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.3279914855957031,
- "epoch": 1.934959349593496,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10826019197702408,
- "kl": 1.3074863090878353e-05,
- "learning_rate": 2.7351693324127037e-07,
- "loss": -0.05556309223175049,
- "num_tokens": 975120.0,
- "reward": 0.64573073387146,
- "reward_std": 0.22739914059638977,
- "rewards/true_env_reward_fn/mean": 0.64573073387146,
- "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 65.52083587646484,
+ "completions/mean_terminated_length": 65.52083587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2291057407855988,
+ "epoch": 5.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09152546525001526,
+ "kl": 0.00013101351214572787,
+ "learning_rate": 4.511238191455491e-07,
+ "loss": 0.043641336262226105,
+ "num_tokens": 5886118.0,
+ "reward": 0.41334670782089233,
+ "reward_std": 0.19480590522289276,
+ "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+ "rewards/true_env_reward_fn/std": 0.19480590522289276,
"step": 238,
- "step_time": 9.220254810001279
+ "step_time": 10.805698846999803
},
{
"clip_ratio/high_max": 0.0,
@@ -6680,26 +6680,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3695034384727478,
- "epoch": 1.943089430894309,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 58.833335876464844,
+ "completions/mean_terminated_length": 58.833335876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3134913444519043,
+ "epoch": 5.829268292682927,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12166028469800949,
- "kl": 2.6563114261080045e-05,
- "learning_rate": 2.6993180912471055e-07,
- "loss": -0.05334407091140747,
- "num_tokens": 980254.0,
- "reward": 0.37203267216682434,
- "reward_std": 0.20089927315711975,
- "rewards/true_env_reward_fn/mean": 0.37203267216682434,
- "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "grad_norm": 0.07343924045562744,
+ "kl": 0.00019655993492051493,
+ "learning_rate": 4.4688912836904533e-07,
+ "loss": 0.015309082344174385,
+ "num_tokens": 5908334.0,
+ "reward": 0.4438478946685791,
+ "reward_std": 0.27188754081726074,
+ "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+ "rewards/true_env_reward_fn/std": 0.27188754081726074,
"step": 239,
- "step_time": 4.224964968001586
+ "step_time": 9.688736522
},
{
"clip_ratio/high_max": 0.0,
@@ -6708,26 +6708,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.75,
- "completions/mean_terminated_length": 53.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.247464120388031,
- "epoch": 1.951219512195122,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2346416860818863,
- "kl": 5.1520751185307745e-05,
- "learning_rate": 2.6635833140104405e-07,
- "loss": -0.0905834436416626,
- "num_tokens": 983244.0,
- "reward": 0.575507640838623,
- "reward_std": 0.17391785979270935,
- "rewards/true_env_reward_fn/mean": 0.575507640838623,
- "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "completions/max_length": 464.0,
+ "completions/max_terminated_length": 464.0,
+ "completions/mean_length": 79.14583587646484,
+ "completions/mean_terminated_length": 79.14583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3148745000362396,
+ "epoch": 5.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06567981839179993,
+ "kl": 4.445325248525478e-05,
+ "learning_rate": 4.4265828730079977e-07,
+ "loss": 0.021218176931142807,
+ "num_tokens": 5929645.0,
+ "reward": 0.5866342186927795,
+ "reward_std": 0.13780196011066437,
+ "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+ "rewards/true_env_reward_fn/std": 0.13780196011066437,
"step": 240,
- "step_time": 3.357481237999309
+ "step_time": 27.046819901000163
},
{
"clip_ratio/high_max": 0.0,
@@ -6736,26 +6736,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2175387144088745,
- "epoch": 1.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1870066374540329,
- "kl": 2.2185965462995227e-05,
- "learning_rate": 2.6279682004185894e-07,
- "loss": -0.07365687191486359,
- "num_tokens": 985574.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 364.0,
+ "completions/max_terminated_length": 364.0,
+ "completions/mean_length": 70.33333587646484,
+ "completions/mean_terminated_length": 70.33333587646484,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2104995846748352,
+ "epoch": 5.878048780487805,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.042626503854990005,
+ "kl": 6.413207393052289e-05,
+ "learning_rate": 4.3843160261067653e-07,
+ "loss": -0.0065308245830237865,
+ "num_tokens": 5950005.0,
+ "reward": 0.5593677759170532,
+ "reward_std": 0.23993276059627533,
+ "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+ "rewards/true_env_reward_fn/std": 0.23993274569511414,
"step": 241,
- "step_time": 4.527591582998866
+ "step_time": 22.238758486000734
},
{
"clip_ratio/high_max": 0.0,
@@ -6764,26 +6764,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.25,
- "completions/mean_terminated_length": 76.25,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2727615237236023,
- "epoch": 1.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.226071804529056e-05,
- "kl": 1.4014385214977665e-05,
- "learning_rate": 2.592475939472668e-07,
- "loss": 7.015369192231447e-07,
- "num_tokens": 988868.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2132116258144379,
+ "epoch": 5.902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05718924477696419,
+ "kl": 6.140609366411809e-05,
+ "learning_rate": 4.342093806672678e-07,
+ "loss": 0.012734346091747284,
+ "num_tokens": 5974626.0,
+ "reward": 0.4349059462547302,
+ "reward_std": 0.24915602803230286,
+ "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+ "rewards/true_env_reward_fn/std": 0.24915601313114166,
"step": 242,
- "step_time": 4.238274277000528
+ "step_time": 9.239750460000323
},
{
"clip_ratio/high_max": 0.0,
@@ -6791,27 +6791,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1519948840141296,
- "epoch": 1.975609756097561,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12447996437549591,
- "kl": 4.700180943473242e-05,
- "learning_rate": 2.557109709173482e-07,
- "loss": 0.08819369971752167,
- "num_tokens": 994068.0,
- "reward": 0.3696666657924652,
- "reward_std": 0.18488828837871552,
- "rewards/true_env_reward_fn/mean": 0.3696666657924652,
- "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 67.04167175292969,
+ "completions/mean_terminated_length": 57.574466705322266,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.7212344706058502,
+ "epoch": 5.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08872146159410477,
+ "kl": 0.00011613740025495645,
+ "learning_rate": 4.2999192751568557e-07,
+ "loss": -0.021363887935876846,
+ "num_tokens": 5997300.0,
+ "reward": 0.4436037540435791,
+ "reward_std": 0.28323379158973694,
+ "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+ "rewards/true_env_reward_fn/std": 0.28323376178741455,
"step": 243,
- "step_time": 3.3367313500002638
+ "step_time": 29.767976787999487
},
{
"clip_ratio/high_max": 0.0,
@@ -6820,26 +6820,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 115.0,
- "completions/max_terminated_length": 115.0,
- "completions/mean_length": 68.25,
- "completions/mean_terminated_length": 68.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0537148416042328,
- "epoch": 1.9837398373983741,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17681685090065002,
- "kl": 3.6240851841284893e-05,
- "learning_rate": 2.521872676236972e-07,
- "loss": 0.05281040072441101,
- "num_tokens": 1000650.0,
- "reward": 0.12209999561309814,
- "reward_std": 0.2502918243408203,
- "rewards/true_env_reward_fn/mean": 0.12209999561309814,
- "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.083335876464844,
+ "completions/mean_terminated_length": 61.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2353634238243103,
+ "epoch": 5.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07658001780509949,
+ "kl": 9.086773115996039e-05,
+ "learning_rate": 4.257795488553798e-07,
+ "loss": -0.005057391710579395,
+ "num_tokens": 6021752.0,
+ "reward": 0.49971556663513184,
+ "reward_std": 0.2643933594226837,
+ "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+ "rewards/true_env_reward_fn/std": 0.2643933594226837,
"step": 244,
- "step_time": 5.2957401019993995
+ "step_time": 9.764708648999658
},
{
"clip_ratio/high_max": 0.0,
@@ -6847,27 +6847,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 140.0,
- "completions/max_terminated_length": 140.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 61.0,
- "completions/min_terminated_length": 61.0,
- "entropy": 1.3126497864723206,
- "epoch": 1.9918699186991868,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16036106646060944,
- "kl": 6.061139720259234e-05,
- "learning_rate": 2.48676799581066e-07,
- "loss": 0.09418506920337677,
- "num_tokens": 1005737.0,
- "reward": 0.31175702810287476,
- "reward_std": 0.38867074251174927,
- "rewards/true_env_reward_fn/mean": 0.31175702810287476,
- "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 86.72917175292969,
+ "completions/mean_terminated_length": 77.68084716796875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2082330882549286,
+ "epoch": 5.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.059102799743413925,
+ "kl": 5.0710960749711376e-05,
+ "learning_rate": 4.215725500179787e-07,
+ "loss": 0.17436102032661438,
+ "num_tokens": 6049747.0,
+ "reward": 0.46306928992271423,
+ "reward_std": 0.26072925329208374,
+ "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+ "rewards/true_env_reward_fn/std": 0.26072925329208374,
"step": 245,
- "step_time": 6.259088058999623
+ "step_time": 31.997988874000384
},
{
"clip_ratio/high_max": 0.0,
@@ -6876,26 +6876,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 48.5,
- "completions/mean_terminated_length": 48.5,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.21333646774292,
- "epoch": 2.0,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23997871577739716,
- "kl": 3.1378609492094256e-05,
- "learning_rate": 2.4517988111911313e-07,
- "loss": 0.010592922568321228,
- "num_tokens": 1010869.0,
- "reward": 0.33381664752960205,
- "reward_std": 0.18213039636611938,
- "rewards/true_env_reward_fn/mean": 0.33381664752960205,
- "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 61.35416793823242,
+ "completions/mean_terminated_length": 61.35416793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2454268038272858,
+ "epoch": 6.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07332625985145569,
+ "kl": 0.0001107546740968246,
+ "learning_rate": 4.1737123594515755e-07,
+ "loss": -0.013426866382360458,
+ "num_tokens": 6072668.0,
+ "reward": 0.45415109395980835,
+ "reward_std": 0.2937946617603302,
+ "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+ "rewards/true_env_reward_fn/std": 0.2937946617603302,
"step": 246,
- "step_time": 3.111915630997828
+ "step_time": 9.587768273000165
},
{
"clip_ratio/high_max": 0.0,
@@ -6904,26 +6904,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.3962982892990112,
- "epoch": 2.008130081300813,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21355785429477692,
- "kl": 3.789625407080166e-05,
- "learning_rate": 2.4169682535425927e-07,
- "loss": 0.025682777166366577,
- "num_tokens": 1014876.0,
- "reward": 0.35749268531799316,
- "reward_std": 0.29738906025886536,
- "rewards/true_env_reward_fn/mean": 0.35749268531799316,
- "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 71.5625,
+ "completions/mean_terminated_length": 71.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1738699078559875,
+ "epoch": 6.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06093747541308403,
+ "kl": 7.893411338955048e-05,
+ "learning_rate": 4.131759111665348e-07,
+ "loss": 0.022837676107883453,
+ "num_tokens": 6095263.0,
+ "reward": 0.5188159346580505,
+ "reward_std": 0.3265886902809143,
+ "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+ "rewards/true_env_reward_fn/std": 0.3265886902809143,
"step": 247,
- "step_time": 3.35338095500083
+ "step_time": 14.232978527999876
},
{
"clip_ratio/high_max": 0.0,
@@ -6932,26 +6932,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 73.875,
- "completions/mean_terminated_length": 73.875,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.463137686252594,
- "epoch": 2.016260162601626,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18188992142677307,
- "kl": 1.746804719005013e-05,
- "learning_rate": 2.382279441616492e-07,
- "loss": -0.17857304215431213,
- "num_tokens": 1018383.0,
- "reward": 0.5329012274742126,
- "reward_std": 0.055823445320129395,
- "rewards/true_env_reward_fn/mean": 0.5329012274742126,
- "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3300949931144714,
+ "epoch": 6.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08098509907722473,
+ "kl": 0.00010828049380506855,
+ "learning_rate": 4.0898687977759887e-07,
+ "loss": 0.01631344109773636,
+ "num_tokens": 6125380.0,
+ "reward": 0.3338983952999115,
+ "reward_std": 0.19050820171833038,
+ "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+ "rewards/true_env_reward_fn/std": 0.19050820171833038,
"step": 248,
- "step_time": 5.210386754000865
+ "step_time": 13.248441182000079
},
{
"clip_ratio/high_max": 0.0,
@@ -6960,26 +6960,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 79.25,
- "completions/mean_terminated_length": 79.25,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.4478936195373535,
- "epoch": 2.024390243902439,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.744662434561178e-05,
- "kl": 1.3336490155779757e-05,
- "learning_rate": 2.3477354814722762e-07,
- "loss": 6.725406365148956e-07,
- "num_tokens": 1022753.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 396.0,
+ "completions/max_terminated_length": 396.0,
+ "completions/mean_length": 88.14583587646484,
+ "completions/mean_terminated_length": 88.14583587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3161276876926422,
+ "epoch": 6.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07784765958786011,
+ "kl": 4.819030800717883e-05,
+ "learning_rate": 4.0480444541766575e-07,
+ "loss": 0.12299837917089462,
+ "num_tokens": 6151955.0,
+ "reward": 0.5446512699127197,
+ "reward_std": 0.2611033618450165,
+ "rewards/true_env_reward_fn/mean": 0.544651210308075,
+ "rewards/true_env_reward_fn/std": 0.2611033618450165,
"step": 249,
- "step_time": 8.798317029002646
+ "step_time": 25.232192139999825
},
{
"clip_ratio/high_max": 0.0,
@@ -6988,32 +6988,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 64.75,
- "completions/mean_terminated_length": 64.75,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.37166029214859,
- "epoch": 2.032520325203252,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 69.97917175292969,
+ "completions/mean_terminated_length": 69.97917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2318329215049744,
+ "epoch": 6.097560975609756,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.1542455554008484,
- "kl": 2.0379054603836266e-05,
- "learning_rate": 2.313339466199264e-07,
- "loss": -0.037539318203926086,
- "num_tokens": 1025971.0,
- "reward": 0.6065863966941833,
- "reward_std": 0.032470256090164185,
- "rewards/true_env_reward_fn/mean": 0.6065863966941833,
- "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "grad_norm": 0.06498592346906662,
+ "kl": 3.854301621686318e-05,
+ "learning_rate": 4.0062891124787e-07,
+ "loss": -0.04302535578608513,
+ "num_tokens": 6174898.0,
+ "reward": 0.6081289052963257,
+ "reward_std": 0.24437586963176727,
+ "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+ "rewards/true_env_reward_fn/std": 0.24437588453292847,
"step": 250,
- "step_time": 4.096263454999644
+ "step_time": 10.385816780999448
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 1025971,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 6174898,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -7028,7 +7028,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-250/training_args.bin b/checkpoint-250/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-250/training_args.bin
+++ b/checkpoint-250/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-300/adapter_model.safetensors b/checkpoint-300/adapter_model.safetensors
index a7bb69e4aa5c7ec9e22ddcdaf0425bdc84f8ef92..332a1ec34dbdb2a460d52f12ad889d3bc25fe07a 100644
--- a/checkpoint-300/adapter_model.safetensors
+++ b/checkpoint-300/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d6714687556a2dc7226afce3096d21be7cf908f496df4aa4046e603946cdae6c
+oid sha256:5cc46d77b2342136f9f964ff8ae019b845ef93695a437d210194a8e468033cea
size 8731128
diff --git a/checkpoint-300/optimizer.pt b/checkpoint-300/optimizer.pt
index 000c07ad367673f21a7b94af8924af5916ddc9ca..836bb3355e935eb182b281494dac07f110f0ff5b 100644
--- a/checkpoint-300/optimizer.pt
+++ b/checkpoint-300/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:8c6d03de482df3140c859a74e70a8a08cb87820c32024dc3ce2c6802304d1ae1
+oid sha256:1b8988bdd91ed6419f5dc399b800fc39bee9c768e2b6f57a4fef63175ca0bdd3
size 17526842
diff --git a/checkpoint-300/ref/adapter_model.safetensors b/checkpoint-300/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-300/ref/adapter_model.safetensors
+++ b/checkpoint-300/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-300/rng_state.pth b/checkpoint-300/rng_state.pth
index 46479067c27a2a4c2fdd57b24455ba6dad5457b6..94e5a8c1adf2b3a3b5e68c40a4c15718fe06cedf 100644
--- a/checkpoint-300/rng_state.pth
+++ b/checkpoint-300/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:12c41259f0bbb68ce4b0c21dedacd5a801851021c681838c318a21744e1672f1
+oid sha256:374987d572ce33eac50f184991a3f10017cfeeedc69adab9b988a899cd74cc76
size 14244
diff --git a/checkpoint-300/scheduler.pt b/checkpoint-300/scheduler.pt
index 8596d3e88a3afc73532e7d6466cf60a9f6b0d26c..f465f6ee477da69c18a63e88eeffa269bbe321dc 100644
--- a/checkpoint-300/scheduler.pt
+++ b/checkpoint-300/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:129a2d7bcd9a56630a9f55dcea61d36958bdd2c2a73a893c6fc4f9afcdcbff14
+oid sha256:23f10e1a2d367b381e414a214dae555e77dac9cb1ab5397f5cb62c7b36b93b56
size 1064
diff --git a/checkpoint-300/trainer_state.json b/checkpoint-300/trainer_state.json
index 51c137478586de02ac26433714bc8fefa956f171..70bfe607e8e09c43a6dd9b63f4590310276321fb 100644
--- a/checkpoint-300/trainer_state.json
+++ b/checkpoint-300/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 2.4390243902439024,
+ "epoch": 7.317073170731708,
"eval_steps": 500,
"global_step": 300,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
},
{
"clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.0,
- "completions/mean_terminated_length": 59.0,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2509461045265198,
- "epoch": 0.8211382113821138,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22887632250785828,
- "kl": 2.1612477212329395e-05,
- "learning_rate": 7.310023360571047e-07,
- "loss": 0.025605827569961548,
- "num_tokens": 414080.0,
- "reward": 0.588032603263855,
- "reward_std": 0.11032751202583313,
- "rewards/true_env_reward_fn/mean": 0.588032603263855,
- "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
"step": 101,
- "step_time": 3.625197022998691
+ "step_time": 16.478299477000064
},
{
"clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 176.0,
- "completions/max_terminated_length": 176.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5443179607391357,
- "epoch": 0.8292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10614532232284546,
- "kl": 1.492139062975184e-05,
- "learning_rate": 7.28862404677924e-07,
- "loss": 0.06531564146280289,
- "num_tokens": 419835.0,
- "reward": 0.07074306160211563,
- "reward_std": 0.2918013632297516,
- "rewards/true_env_reward_fn/mean": 0.07074306160211563,
- "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
"step": 102,
- "step_time": 7.796810614998321
+ "step_time": 11.344593008000174
},
{
"clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 101.0,
"completions/max_terminated_length": 101.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3223788738250732,
- "epoch": 0.8373983739837398,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2421368807554245,
- "kl": 3.265505938543356e-05,
- "learning_rate": 7.266930267388503e-07,
- "loss": -0.07752113044261932,
- "num_tokens": 422773.0,
- "reward": 0.33568501472473145,
- "reward_std": 0.2780380845069885,
- "rewards/true_env_reward_fn/mean": 0.33568501472473145,
- "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
"step": 103,
- "step_time": 4.313938073000827
+ "step_time": 8.135681302999728
},
{
"clip_ratio/high_max": 0.0,
@@ -2900,26 +2900,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 67.25,
- "completions/mean_terminated_length": 67.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.3332037329673767,
- "epoch": 0.8455284552845529,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13883370161056519,
- "kl": 2.1224042484391248e-05,
- "learning_rate": 7.244943964874369e-07,
- "loss": 0.021739646792411804,
- "num_tokens": 426507.0,
- "reward": 0.40595096349716187,
- "reward_std": 0.2035457342863083,
- "rewards/true_env_reward_fn/mean": 0.40595096349716187,
- "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
"step": 104,
- "step_time": 4.155937195999286
+ "step_time": 15.185034105999875
},
{
"clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
"completions/min_length": 31.0,
"completions/min_terminated_length": 31.0,
- "entropy": 1.3391229510307312,
- "epoch": 0.8536585365853658,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24205009639263153,
- "kl": 2.5022183763212524e-05,
- "learning_rate": 7.222667107905085e-07,
- "loss": 0.06330433487892151,
- "num_tokens": 429010.0,
- "reward": 0.3355163037776947,
- "reward_std": 0.2902730703353882,
- "rewards/true_env_reward_fn/mean": 0.3355163037776947,
- "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
"step": 105,
- "step_time": 3.808478789000219
+ "step_time": 9.588520330999927
},
{
"clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1864720582962036,
- "epoch": 0.8617886178861789,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13473568856716156,
- "kl": 1.4212585938366828e-05,
- "learning_rate": 7.200101691165338e-07,
- "loss": -0.020715661346912384,
- "num_tokens": 432403.0,
- "reward": 0.4871198534965515,
- "reward_std": 0.15407639741897583,
- "rewards/true_env_reward_fn/mean": 0.4871198534965515,
- "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
"step": 106,
- "step_time": 4.240638332001254
+ "step_time": 14.60399662399982
},
{
"clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.0,
- "completions/mean_terminated_length": 48.0,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "entropy": 1.0669284462928772,
- "epoch": 0.8699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14131899178028107,
- "kl": 1.5787159554747632e-05,
- "learning_rate": 7.177249735177651e-07,
- "loss": 0.03678784519433975,
- "num_tokens": 435995.0,
- "reward": 0.5010770559310913,
- "reward_std": 0.48966261744499207,
- "rewards/true_env_reward_fn/mean": 0.5010770559310913,
- "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
"step": 107,
- "step_time": 3.3587191269998584
+ "step_time": 20.100954443999854
},
{
"clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 71.875,
- "completions/mean_terminated_length": 71.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.304731547832489,
- "epoch": 0.8780487804878049,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.380985673284158e-05,
- "kl": 1.3128728824085556e-05,
- "learning_rate": 7.154113286121462e-07,
- "loss": 6.494262834166875e-07,
- "num_tokens": 442094.0,
- "reward": 0.4055500030517578,
- "reward_std": 0.052258480340242386,
- "rewards/true_env_reward_fn/mean": 0.4055500030517578,
- "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
"step": 108,
- "step_time": 4.337008413998774
+ "step_time": 8.304149297999857
},
{
"clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2324069738388062,
- "epoch": 0.8861788617886179,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13588950037956238,
- "kl": 1.3448377558233915e-05,
- "learning_rate": 7.130694415649912e-07,
- "loss": 1.0952353477478027e-06,
- "num_tokens": 447226.0,
- "reward": 0.20854972302913666,
- "reward_std": 0.06059705466032028,
- "rewards/true_env_reward_fn/mean": 0.20854972302913666,
- "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
"step": 109,
- "step_time": 3.2976038649994734
+ "step_time": 11.260021517000041
},
{
"clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1567262411117554,
- "epoch": 0.8943089430894309,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.736967720324174e-05,
- "kl": 1.2838129805459175e-05,
- "learning_rate": 7.106995220704342e-07,
- "loss": 6.425898391171359e-07,
- "num_tokens": 450359.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
"step": 110,
- "step_time": 4.067084037998939
+ "step_time": 22.80178854000019
},
{
"clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 100.0,
"completions/max_terminated_length": 100.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 12.0,
- "completions/min_terminated_length": 12.0,
- "entropy": 1.496058464050293,
- "epoch": 0.9024390243902439,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18041981756687164,
- "kl": 1.6616825632809196e-05,
- "learning_rate": 7.083017823326532e-07,
- "loss": 0.0269068144261837,
- "num_tokens": 453583.0,
- "reward": 0.5647265911102295,
- "reward_std": 0.1507105529308319,
- "rewards/true_env_reward_fn/mean": 0.5647265911102295,
- "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
"step": 111,
- "step_time": 4.347732382997492
+ "step_time": 9.663163859000178
},
{
"clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 67.75,
- "completions/mean_terminated_length": 67.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.4196155667304993,
- "epoch": 0.9105691056910569,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18451112508773804,
- "kl": 2.1803500203532167e-05,
- "learning_rate": 7.058764370468698e-07,
- "loss": 0.1650262475013733,
- "num_tokens": 456773.0,
- "reward": 0.6907394528388977,
- "reward_std": 0.1393815129995346,
- "rewards/true_env_reward_fn/mean": 0.6907394528388977,
- "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
"step": 112,
- "step_time": 4.627644968999448
+ "step_time": 9.903081222999617
},
{
"clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2243221998214722,
- "epoch": 0.9186991869918699,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331371545791626,
- "kl": 1.548633599668392e-05,
- "learning_rate": 7.034237033801247e-07,
- "loss": 0.039844345301389694,
- "num_tokens": 462714.0,
- "reward": 0.21676866710186005,
- "reward_std": 0.26559779047966003,
- "rewards/true_env_reward_fn/mean": 0.21676866710186005,
- "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
"step": 113,
- "step_time": 3.8455466220002563
+ "step_time": 10.101770388999284
},
{
"clip_ratio/high_max": 0.0,
@@ -3180,54 +3180,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 73.0,
- "completions/mean_terminated_length": 73.0,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.325823724269867,
- "epoch": 0.926829268292683,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1866220384836197,
- "kl": 1.8801019905367866e-05,
- "learning_rate": 7.009438009518325e-07,
- "loss": 0.06504581868648529,
- "num_tokens": 465994.0,
- "reward": 0.5194582939147949,
- "reward_std": 0.2796703577041626,
- "rewards/true_env_reward_fn/mean": 0.5194582939147949,
- "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
"step": 114,
- "step_time": 4.151028698999653
+ "step_time": 17.633509853000305
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
- "clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1521879434585571,
- "epoch": 0.9349593495934959,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14320029318332672,
- "kl": 1.2749982033710694e-05,
- "learning_rate": 6.98436951814117e-07,
- "loss": 0.03685300797224045,
- "num_tokens": 468615.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
"step": 115,
- "step_time": 3.6973990600017714
+ "step_time": 12.307247350000125
},
{
"clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.201507806777954,
- "epoch": 0.943089430894309,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14264807105064392,
- "kl": 2.6679515940486453e-05,
- "learning_rate": 6.959033804319283e-07,
- "loss": -0.023484818637371063,
- "num_tokens": 471647.0,
- "reward": 0.41836902499198914,
- "reward_std": 0.3116860091686249,
- "rewards/true_env_reward_fn/mean": 0.41836902499198914,
- "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
"step": 116,
- "step_time": 3.1295652919998247
+ "step_time": 14.468690254000194
},
{
"clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 62.375,
- "completions/mean_terminated_length": 62.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2834057807922363,
- "epoch": 0.9512195121951219,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.810227154754102e-05,
- "kl": 1.4841665233689127e-05,
- "learning_rate": 6.933433136629443e-07,
- "loss": 7.425555850204546e-07,
- "num_tokens": 474682.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
"step": 117,
- "step_time": 3.4368692790012574
+ "step_time": 11.849063975999798
},
{
"clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.191932499408722,
- "epoch": 0.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21204856038093567,
- "kl": 3.64198385796044e-05,
- "learning_rate": 6.907569807372574e-07,
- "loss": -0.001312553882598877,
- "num_tokens": 477027.0,
- "reward": 0.5300568342208862,
- "reward_std": 0.2945883274078369,
- "rewards/true_env_reward_fn/mean": 0.5300568342208862,
- "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
"step": 118,
- "step_time": 3.8569856240010267
+ "step_time": 16.597334930000216
},
{
"clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 190.0,
- "completions/max_terminated_length": 190.0,
- "completions/mean_length": 96.5,
- "completions/mean_terminated_length": 96.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2401175498962402,
- "epoch": 0.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011377666669432074,
- "kl": 1.3742283954343293e-05,
- "learning_rate": 6.881446132368494e-07,
- "loss": 6.866695230201003e-07,
- "num_tokens": 481999.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
"step": 119,
- "step_time": 8.09440958399864
+ "step_time": 11.729475523999554
},
{
"clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.230682611465454,
- "epoch": 0.975609756097561,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22175048291683197,
- "kl": 1.2522132237791084e-05,
- "learning_rate": 6.855064450748555e-07,
- "loss": -0.04083740711212158,
- "num_tokens": 490884.0,
- "reward": 0.13476666808128357,
- "reward_std": 0.2987530529499054,
- "rewards/true_env_reward_fn/mean": 0.13476666808128357,
- "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
"step": 120,
- "step_time": 4.678523641001448
+ "step_time": 11.503627788000358
},
{
"clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2799639105796814,
- "epoch": 0.983739837398374,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19926966726779938,
- "kl": 1.7022688552970067e-05,
- "learning_rate": 6.828427124746189e-07,
- "loss": -0.010804429650306702,
- "num_tokens": 496404.0,
- "reward": 0.24633333086967468,
- "reward_std": 0.2454334795475006,
- "rewards/true_env_reward_fn/mean": 0.24633333086967468,
- "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
"step": 121,
- "step_time": 3.98071062300005
+ "step_time": 8.728293746000418
},
{
"clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 46.875,
- "completions/mean_terminated_length": 46.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3840235471725464,
- "epoch": 0.991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24853822588920593,
- "kl": 3.688259130285587e-05,
- "learning_rate": 6.801536539485403e-07,
- "loss": 0.10205884277820587,
- "num_tokens": 499767.0,
- "reward": 0.3045905530452728,
- "reward_std": 0.262839138507843,
- "rewards/true_env_reward_fn/mean": 0.3045905530452728,
- "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
"step": 122,
- "step_time": 3.3792565210005705
+ "step_time": 15.51957702100026
},
{
"clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 50.375,
- "completions/mean_terminated_length": 50.375,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2064164280891418,
- "epoch": 1.0,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1365528553724289,
- "kl": 2.434901080050622e-05,
- "learning_rate": 6.774395102767203e-07,
- "loss": -0.03472680225968361,
- "num_tokens": 504906.0,
- "reward": 0.2722649872303009,
- "reward_std": 0.2922348082065582,
- "rewards/true_env_reward_fn/mean": 0.2722649872303009,
- "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
"step": 123,
- "step_time": 3.0233660449994204
+ "step_time": 8.471463828000196
},
{
"clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2819936871528625,
- "epoch": 1.008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00023045104171615094,
- "kl": 2.2608143808611203e-05,
- "learning_rate": 6.747005244854004e-07,
- "loss": 1.1284330412308918e-06,
- "num_tokens": 508329.0,
- "reward": 0.3149532079696655,
- "reward_std": 0.4275679290294647,
- "rewards/true_env_reward_fn/mean": 0.3149532079696655,
- "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
"step": 124,
- "step_time": 4.01701365199915
+ "step_time": 13.438758649999727
},
{
"clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 157.0,
- "completions/max_terminated_length": 157.0,
- "completions/mean_length": 80.875,
- "completions/mean_terminated_length": 80.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1542360186576843,
- "epoch": 1.016260162601626,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.232049003709108e-05,
- "kl": 1.291002809011843e-05,
- "learning_rate": 6.719369418252023e-07,
- "loss": 6.488799613180163e-07,
- "num_tokens": 515076.0,
- "reward": 0.4841846525669098,
- "reward_std": 0.12780573964118958,
- "rewards/true_env_reward_fn/mean": 0.4841846525669098,
- "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
"step": 125,
- "step_time": 7.240956699999515
+ "step_time": 8.654177170999901
},
{
"clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2594389915466309,
- "epoch": 1.024390243902439,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13091468811035156,
- "kl": 1.751603304001037e-05,
- "learning_rate": 6.691490097491675e-07,
- "loss": -0.033413223922252655,
- "num_tokens": 520279.0,
- "reward": 0.28095200657844543,
- "reward_std": 0.21837711334228516,
- "rewards/true_env_reward_fn/mean": 0.28095200657844543,
- "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
"step": 126,
- "step_time": 3.355879656997786
+ "step_time": 10.513378469000145
},
{
"clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 136.0,
- "completions/max_terminated_length": 136.0,
- "completions/mean_length": 77.875,
- "completions/mean_terminated_length": 77.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.3044686317443848,
- "epoch": 1.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12953205406665802,
- "kl": 1.8700401597016025e-05,
- "learning_rate": 6.663369778906008e-07,
- "loss": 0.03562816232442856,
- "num_tokens": 524582.0,
- "reward": 0.4330660402774811,
- "reward_std": 0.4592672288417816,
- "rewards/true_env_reward_fn/mean": 0.4330660402774811,
- "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
"step": 127,
- "step_time": 5.965807722999671
+ "step_time": 23.20779430600078
},
{
"clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 117.0,
- "completions/max_terminated_length": 117.0,
- "completions/mean_length": 75.375,
- "completions/mean_terminated_length": 75.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1742327809333801,
- "epoch": 1.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14993594586849213,
- "kl": 1.1459212601039326e-05,
- "learning_rate": 6.635010980407174e-07,
- "loss": 0.03646668791770935,
- "num_tokens": 526213.0,
- "reward": 0.7185037136077881,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.7185037136077881,
- "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
"step": 128,
- "step_time": 4.9305356690001645
+ "step_time": 11.551069149999876
},
{
"clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 49.125,
- "completions/mean_terminated_length": 49.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0784690976142883,
- "epoch": 1.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16007214784622192,
- "kl": 1.2491957932070363e-05,
- "learning_rate": 6.606416241260979e-07,
- "loss": 0.006608985364437103,
- "num_tokens": 531862.0,
- "reward": 0.2934249937534332,
- "reward_std": 0.2395382523536682,
- "rewards/true_env_reward_fn/mean": 0.2934249937534332,
- "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
"step": 129,
- "step_time": 3.173622508000335
+ "step_time": 10.723005456000465
},
{
"clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.308219850063324,
- "epoch": 1.056910569105691,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.985446427483112e-05,
- "kl": 1.2420873190421844e-05,
- "learning_rate": 6.577588121859508e-07,
- "loss": 6.241918413252279e-07,
- "num_tokens": 535957.0,
- "reward": 0.4817493259906769,
- "reward_std": 0.029202036559581757,
- "rewards/true_env_reward_fn/mean": 0.4817493259906769,
- "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
"step": 130,
- "step_time": 4.251137947001553
+ "step_time": 15.15810051499966
},
{
"clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.0767641067504883,
- "epoch": 1.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15786457061767578,
- "kl": 1.8847958926926367e-05,
- "learning_rate": 6.548529203491875e-07,
- "loss": -0.0026272237300872803,
- "num_tokens": 539269.0,
- "reward": 0.536803662776947,
- "reward_std": 0.30375123023986816,
- "rewards/true_env_reward_fn/mean": 0.536803662776947,
- "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
"step": 131,
- "step_time": 3.7980547870010923
+ "step_time": 20.109428818001106
},
{
"clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3295028805732727,
- "epoch": 1.0731707317073171,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1881481558084488,
- "kl": 2.0969039724150207e-05,
- "learning_rate": 6.519242088113085e-07,
- "loss": 0.08431969583034515,
- "num_tokens": 545691.0,
- "reward": 0.24590599536895752,
- "reward_std": 0.2047487199306488,
- "rewards/true_env_reward_fn/mean": 0.24590599536895752,
- "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
"step": 132,
- "step_time": 4.361092664001262
+ "step_time": 10.12403799699996
},
{
"clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2131375670433044,
- "epoch": 1.08130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13930389285087585,
- "kl": 1.1046585314034019e-05,
- "learning_rate": 6.489729398111058e-07,
- "loss": -0.03801802545785904,
- "num_tokens": 550295.0,
- "reward": 0.3215479254722595,
- "reward_std": 0.1736886352300644,
- "rewards/true_env_reward_fn/mean": 0.3215479254722595,
- "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
"step": 133,
- "step_time": 3.372364626999115
+ "step_time": 10.396350653999434
},
{
"clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 63.375,
- "completions/mean_terminated_length": 63.375,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2786019444465637,
- "epoch": 1.089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12932609021663666,
- "kl": 1.340499647994875e-05,
- "learning_rate": 6.459993776071815e-07,
- "loss": 0.029022663831710815,
- "num_tokens": 553826.0,
- "reward": 0.4830188751220703,
- "reward_std": 0.29014864563941956,
- "rewards/true_env_reward_fn/mean": 0.4830188751220703,
- "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
"step": 134,
- "step_time": 3.215292060998763
+ "step_time": 10.961974539000039
},
{
"clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 0.9003906548023224,
- "epoch": 1.0975609756097562,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15958240628242493,
- "kl": 2.838099044311093e-05,
- "learning_rate": 6.430037884542861e-07,
- "loss": 0.11459673941135406,
- "num_tokens": 557217.0,
- "reward": 0.494448184967041,
- "reward_std": 0.3076546788215637,
- "rewards/true_env_reward_fn/mean": 0.494448184967041,
- "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
"step": 135,
- "step_time": 3.500462582000182
+ "step_time": 11.779171687000144
},
{
"clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 93.0,
"completions/max_terminated_length": 93.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2928712964057922,
- "epoch": 1.1056910569105691,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23077522218227386,
- "kl": 2.251418845844455e-05,
- "learning_rate": 6.399864405794782e-07,
- "loss": -0.05874824523925781,
- "num_tokens": 562421.0,
- "reward": 0.2385583370923996,
- "reward_std": 0.23380905389785767,
- "rewards/true_env_reward_fn/mean": 0.2385583370923996,
- "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
"step": 136,
- "step_time": 4.208805245998519
+ "step_time": 8.779588141000204
},
{
"clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8736326098442078,
- "epoch": 1.113821138211382,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.871674071997404e-05,
- "kl": 1.1485328741400735e-05,
- "learning_rate": 6.369476041581066e-07,
- "loss": 5.747077125306532e-07,
- "num_tokens": 566387.0,
- "reward": 0.4902166724205017,
- "reward_std": 0.038254011422395706,
- "rewards/true_env_reward_fn/mean": 0.4902166724205017,
- "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
"step": 137,
- "step_time": 3.981489739000608
+ "step_time": 11.74765996799988
},
{
"clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1538971662521362,
- "epoch": 1.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14676779508590698,
- "kl": 1.1651037766569061e-05,
- "learning_rate": 6.338875512896188e-07,
- "loss": 0.1347643882036209,
- "num_tokens": 569341.0,
- "reward": 0.43844783306121826,
- "reward_std": 0.16067014634609222,
- "rewards/true_env_reward_fn/mean": 0.43844783306121826,
- "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
"step": 138,
- "step_time": 3.689221037999232
+ "step_time": 8.8135579650002
},
{
"clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1961707472801208,
- "epoch": 1.1300813008130082,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16650564968585968,
- "kl": 1.4349476259667426e-05,
- "learning_rate": 6.308065559731976e-07,
- "loss": 0.007910434156656265,
- "num_tokens": 574046.0,
- "reward": 0.4596000015735626,
- "reward_std": 0.07715634256601334,
- "rewards/true_env_reward_fn/mean": 0.4596000015735626,
- "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
"step": 139,
- "step_time": 3.6711935700004688
+ "step_time": 9.997661417000472
},
{
"clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2013322114944458,
- "epoch": 1.1382113821138211,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14415834844112396,
- "kl": 1.4664098671346437e-05,
- "learning_rate": 6.277048940832264e-07,
- "loss": -0.016162052750587463,
- "num_tokens": 576769.0,
- "reward": 0.6152583360671997,
- "reward_std": 0.07727260142564774,
- "rewards/true_env_reward_fn/mean": 0.6152583360671997,
- "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
"step": 140,
- "step_time": 3.5191362610003125
+ "step_time": 14.679971276999822
},
{
"clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.75,
- "completions/mean_terminated_length": 44.75,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0287770330905914,
- "epoch": 1.146341463414634,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.892051457427442e-05,
- "kl": 1.1797974821092794e-05,
- "learning_rate": 6.245828433445872e-07,
- "loss": 5.92092192164273e-07,
- "num_tokens": 578843.0,
- "reward": 0.6387845277786255,
- "reward_std": 0.13867565989494324,
- "rewards/true_env_reward_fn/mean": 0.6387845277786255,
- "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
"step": 141,
- "step_time": 2.5015027329991426
+ "step_time": 11.64747691499997
},
{
"clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1590029001235962,
- "epoch": 1.1544715447154472,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.160966694355011,
- "kl": 1.4735675904375967e-05,
- "learning_rate": 6.214406833077937e-07,
- "loss": 0.0170527845621109,
- "num_tokens": 583201.0,
- "reward": 0.36017733812332153,
- "reward_std": 0.3556094467639923,
- "rewards/true_env_reward_fn/mean": 0.36017733812332153,
- "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
"step": 142,
- "step_time": 3.2783409929998015
+ "step_time": 8.380270293999729
},
{
"clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 59.0,
- "completions/min_terminated_length": 59.0,
- "entropy": 1.1985241174697876,
- "epoch": 1.1626016260162602,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12272457778453827,
- "kl": 1.7849098185251933e-05,
- "learning_rate": 6.182786953239593e-07,
- "loss": -0.0016125142574310303,
- "num_tokens": 587317.0,
- "reward": 0.34745320677757263,
- "reward_std": 0.3954337239265442,
- "rewards/true_env_reward_fn/mean": 0.34745320677757263,
- "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
"step": 143,
- "step_time": 3.9932043310000154
+ "step_time": 11.716556537000088
},
{
"clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 48.0,
- "completions/max_terminated_length": 48.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1116944551467896,
- "epoch": 1.170731707317073,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18435250222682953,
- "kl": 1.014559029499651e-05,
- "learning_rate": 6.150971625196048e-07,
- "loss": 0.009793907403945923,
- "num_tokens": 590191.0,
- "reward": 0.4938516616821289,
- "reward_std": 0.03703190013766289,
- "rewards/true_env_reward_fn/mean": 0.4938516616821289,
- "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
"step": 144,
- "step_time": 2.3663663690022076
+ "step_time": 10.67718802499985
},
{
"clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.0,
- "completions/mean_terminated_length": 53.0,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.413200855255127,
- "epoch": 1.1788617886178863,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14336225390434265,
- "kl": 2.0541991034406237e-05,
- "learning_rate": 6.118963697713078e-07,
- "loss": -0.013927727937698364,
- "num_tokens": 593671.0,
- "reward": 0.4619143605232239,
- "reward_std": 0.3773181140422821,
- "rewards/true_env_reward_fn/mean": 0.4619143605232239,
- "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
"step": 145,
- "step_time": 3.9730388410007436
+ "step_time": 8.610043666999445
},
{
"clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 128.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 6.0,
- "completions/min_terminated_length": 6.0,
- "entropy": 1.08676016330719,
- "epoch": 1.1869918699186992,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15555191040039062,
- "kl": 1.6947700260061538e-05,
- "learning_rate": 6.086766036801937e-07,
- "loss": -0.139797180891037,
- "num_tokens": 601612.0,
- "reward": 0.3831036686897278,
- "reward_std": 0.09242849797010422,
- "rewards/true_env_reward_fn/mean": 0.3831036686897278,
- "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
"step": 146,
- "step_time": 6.323679949000507
+ "step_time": 13.649344002000362
},
{
"clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.5055813789367676,
- "epoch": 1.1951219512195121,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2397669553756714,
- "kl": 2.704876442294335e-05,
- "learning_rate": 6.054381525462745e-07,
- "loss": 0.2738838493824005,
- "num_tokens": 606712.0,
- "reward": 0.25339600443840027,
- "reward_std": 0.3023079037666321,
- "rewards/true_env_reward_fn/mean": 0.25339600443840027,
- "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
"step": 147,
- "step_time": 5.185072233998653
+ "step_time": 11.702765863000877
},
{
"clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 47.25,
- "completions/mean_terminated_length": 47.25,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.135968267917633,
- "epoch": 1.203252032520325,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2614514231681824,
- "kl": 3.613240005506668e-05,
- "learning_rate": 6.021813063426323e-07,
- "loss": 0.10286401212215424,
- "num_tokens": 610566.0,
- "reward": 0.31031692028045654,
- "reward_std": 0.3124054968357086,
- "rewards/true_env_reward_fn/mean": 0.31031692028045654,
- "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
"step": 148,
- "step_time": 3.2177847610000754
+ "step_time": 13.306644664000487
},
{
"clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 52.75,
- "completions/mean_terminated_length": 52.75,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4589928984642029,
- "epoch": 1.2113821138211383,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2019941508769989,
- "kl": 2.1841721718374174e-05,
- "learning_rate": 5.989063566894572e-07,
- "loss": 0.010915875434875488,
- "num_tokens": 615716.0,
- "reward": 0.31711751222610474,
- "reward_std": 0.13289952278137207,
- "rewards/true_env_reward_fn/mean": 0.31711751222610474,
- "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
"step": 149,
- "step_time": 4.3804878079990885
+ "step_time": 9.446422488999815
},
{
"clip_ratio/high_max": 0.0,
@@ -4188,26 +4188,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.1892729997634888,
- "epoch": 1.2195121951219512,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.835455471649766e-05,
- "kl": 1.3420096820482286e-05,
- "learning_rate": 5.956135968279332e-07,
- "loss": 6.646802717114042e-07,
- "num_tokens": 619439.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
"step": 150,
- "step_time": 3.63938895299907
+ "step_time": 10.538116119999813
},
{
"clip_ratio/high_max": 0.0,
@@ -4216,26 +4216,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1402934789657593,
- "epoch": 1.2276422764227641,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010795716661959887,
- "kl": 1.4652535810455447e-05,
- "learning_rate": 5.923033215939834e-07,
- "loss": 7.542968205598299e-07,
- "num_tokens": 621009.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
"step": 151,
- "step_time": 3.926544339999964
+ "step_time": 15.755764130999978
},
{
"clip_ratio/high_max": 0.0,
@@ -4244,26 +4244,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
"completions/mean_length": 64.125,
"completions/mean_terminated_length": 64.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.190350890159607,
- "epoch": 1.2357723577235773,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19853363931179047,
- "kl": 2.269768037876929e-05,
- "learning_rate": 5.889758273918683e-07,
- "loss": 0.044217392802238464,
- "num_tokens": 623994.0,
- "reward": 0.4411996603012085,
- "reward_std": 0.2517909109592438,
- "rewards/true_env_reward_fn/mean": 0.4411996603012085,
- "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
"step": 152,
- "step_time": 3.7339736520007136
+ "step_time": 9.670861957999932
},
{
"clip_ratio/high_max": 0.0,
@@ -4272,26 +4272,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2316884994506836,
- "epoch": 1.2439024390243902,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.35867181699723e-05,
- "kl": 1.1579370038816705e-05,
- "learning_rate": 5.856314121676467e-07,
- "loss": 5.79387460675207e-07,
- "num_tokens": 628224.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
"step": 153,
- "step_time": 4.375236807001784
+ "step_time": 22.82054339000024
},
{
"clip_ratio/high_max": 0.0,
@@ -4300,26 +4300,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4028943181037903,
- "epoch": 1.2520325203252032,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25418931245803833,
- "kl": 3.6890452065563295e-05,
- "learning_rate": 5.822703753824966e-07,
- "loss": 0.25599968433380127,
- "num_tokens": 631183.0,
- "reward": 0.38683533668518066,
- "reward_std": 0.43613559007644653,
- "rewards/true_env_reward_fn/mean": 0.38683533668518066,
- "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
"step": 154,
- "step_time": 6.055355972999678
+ "step_time": 9.973958625999785
},
{
"clip_ratio/high_max": 0.0,
@@ -4328,26 +4328,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.367663562297821,
- "epoch": 1.2601626016260163,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331811130046844,
- "kl": 1.2863993106293492e-05,
- "learning_rate": 5.788930179859024e-07,
- "loss": -0.0047044456005096436,
- "num_tokens": 636230.0,
- "reward": 0.3489508628845215,
- "reward_std": 0.13627417385578156,
- "rewards/true_env_reward_fn/mean": 0.3489508628845215,
- "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
"step": 155,
- "step_time": 3.9174396130001696
+ "step_time": 9.758407419000378
},
{
"clip_ratio/high_max": 0.0,
@@ -4356,26 +4356,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 135.0,
- "completions/max_terminated_length": 135.0,
- "completions/mean_length": 68.5,
- "completions/mean_terminated_length": 68.5,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.2759611010551453,
- "epoch": 1.2682926829268293,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12738144397735596,
- "kl": 1.5844128029129934e-05,
- "learning_rate": 5.754996423887061e-07,
- "loss": -0.011055335402488708,
- "num_tokens": 640262.0,
- "reward": 0.344404935836792,
- "reward_std": 0.16542991995811462,
- "rewards/true_env_reward_fn/mean": 0.344404935836792,
- "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
"step": 156,
- "step_time": 5.706334413998775
+ "step_time": 11.009583763999672
},
{
"clip_ratio/high_max": 0.0,
@@ -4384,26 +4384,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.0904476642608643,
- "epoch": 1.2764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.733699021628127e-05,
- "kl": 1.2238857834745431e-05,
- "learning_rate": 5.720905524360308e-07,
- "loss": 6.076299996493617e-07,
- "num_tokens": 645091.0,
- "reward": 0.4731999933719635,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4731999933719635,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
"step": 157,
- "step_time": 3.7696847109982627
+ "step_time": 10.12789283499933
},
{
"clip_ratio/high_max": 0.0,
@@ -4412,26 +4412,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.0,
- "completions/mean_terminated_length": 60.0,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.3856677412986755,
- "epoch": 1.2845528455284554,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18670028448104858,
- "kl": 1.8415606064081658e-05,
- "learning_rate": 5.686660533800736e-07,
- "loss": -0.07078710198402405,
- "num_tokens": 648179.0,
- "reward": 0.537517786026001,
- "reward_std": 0.1451217085123062,
- "rewards/true_env_reward_fn/mean": 0.537517786026001,
- "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
"step": 158,
- "step_time": 3.7075291149994882
+ "step_time": 7.800485663000472
},
{
"clip_ratio/high_max": 0.0,
@@ -4440,26 +4440,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.125,
- "completions/mean_terminated_length": 68.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.121916651725769,
- "epoch": 1.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11415883898735046,
- "kl": 1.909901220642496e-05,
- "learning_rate": 5.652264518527725e-07,
- "loss": -0.04401372745633125,
- "num_tokens": 652044.0,
- "reward": 0.5182899832725525,
- "reward_std": 0.21869486570358276,
- "rewards/true_env_reward_fn/mean": 0.5182899832725525,
- "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
"step": 159,
- "step_time": 3.8929355969994504
+ "step_time": 13.77649076500029
},
{
"clip_ratio/high_max": 0.0,
@@ -4468,26 +4468,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 143.0,
- "completions/max_terminated_length": 143.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5205118060112,
- "epoch": 1.3008130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2403375506401062,
- "kl": 3.7574073758150917e-05,
- "learning_rate": 5.617720558383508e-07,
- "loss": 0.26385918259620667,
- "num_tokens": 656362.0,
- "reward": 0.41201668977737427,
- "reward_std": 0.2023741453886032,
- "rewards/true_env_reward_fn/mean": 0.41201668977737427,
- "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
"step": 160,
- "step_time": 6.157555950998358
+ "step_time": 13.25029361300085
},
{
"clip_ratio/high_max": 0.0,
@@ -4496,26 +4496,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3556928038597107,
- "epoch": 1.3089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18135924637317657,
- "kl": 2.6372636057203636e-05,
- "learning_rate": 5.583031746457407e-07,
- "loss": -0.10538280755281448,
- "num_tokens": 659977.0,
- "reward": 0.4239906072616577,
- "reward_std": 0.3287450969219208,
- "rewards/true_env_reward_fn/mean": 0.4239906072616577,
- "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
"step": 161,
- "step_time": 4.709477423999488
+ "step_time": 17.837881629000094
},
{
"clip_ratio/high_max": 0.0,
@@ -4524,26 +4524,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 52.5,
- "completions/mean_terminated_length": 52.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 0.9743769466876984,
- "epoch": 1.3170731707317074,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.178797647356987,
- "kl": 1.2532927030406427e-05,
- "learning_rate": 5.548201188808869e-07,
- "loss": -0.04164513945579529,
- "num_tokens": 661409.0,
- "reward": 0.8041956424713135,
- "reward_std": 0.1363772451877594,
- "rewards/true_env_reward_fn/mean": 0.8041956424713135,
- "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
"step": 162,
- "step_time": 3.1791253910014348
+ "step_time": 9.286757633000889
},
{
"clip_ratio/high_max": 0.0,
@@ -4552,26 +4552,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.3072250485420227,
- "epoch": 1.3252032520325203,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16768279671669006,
- "kl": 1.89386219062726e-05,
- "learning_rate": 5.513232004189339e-07,
- "loss": -0.01292814314365387,
- "num_tokens": 666504.0,
- "reward": 0.27981066703796387,
- "reward_std": 0.2949208915233612,
- "rewards/true_env_reward_fn/mean": 0.27981066703796387,
- "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
"step": 163,
- "step_time": 3.778431355000066
+ "step_time": 10.785562561999996
},
{
"clip_ratio/high_max": 0.0,
@@ -4580,26 +4580,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 51.0,
- "completions/mean_terminated_length": 51.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1984856128692627,
- "epoch": 1.3333333333333333,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13123764097690582,
- "kl": 1.9091786271019373e-05,
- "learning_rate": 5.478127323763027e-07,
- "loss": 0.035523779690265656,
- "num_tokens": 671004.0,
- "reward": 0.2771500051021576,
- "reward_std": 0.30146247148513794,
- "rewards/true_env_reward_fn/mean": 0.2771500051021576,
- "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
"step": 164,
- "step_time": 3.304021460000513
+ "step_time": 12.722446307000155
},
{
"clip_ratio/high_max": 0.0,
@@ -4608,26 +4608,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 102.0,
- "completions/max_terminated_length": 102.0,
- "completions/mean_length": 84.875,
- "completions/mean_terminated_length": 84.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2726752758026123,
- "epoch": 1.3414634146341464,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15594874322414398,
- "kl": 2.5981638827943243e-05,
- "learning_rate": 5.442890290826518e-07,
- "loss": -0.01398652046918869,
- "num_tokens": 677307.0,
- "reward": 0.35573017597198486,
- "reward_std": 0.25944042205810547,
- "rewards/true_env_reward_fn/mean": 0.35573017597198486,
- "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
"step": 165,
- "step_time": 4.972808451999299
+ "step_time": 14.89655208000022
},
{
"clip_ratio/high_max": 0.0,
@@ -4636,26 +4636,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 210.0,
- "completions/max_terminated_length": 210.0,
- "completions/mean_length": 103.625,
- "completions/mean_terminated_length": 103.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0949090719223022,
- "epoch": 1.3495934959349594,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.07191785424947739,
- "kl": 1.4828182884230046e-05,
- "learning_rate": 5.407524060527332e-07,
- "loss": -0.05063021928071976,
- "num_tokens": 682076.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
"step": 166,
- "step_time": 8.769379133000257
+ "step_time": 8.962532588000613
},
{
"clip_ratio/high_max": 0.0,
@@ -4664,26 +4664,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1857684254646301,
- "epoch": 1.3577235772357723,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.923542191041633e-05,
- "kl": 1.0165251751459436e-05,
- "learning_rate": 5.37203179958141e-07,
- "loss": 5.114516170579009e-07,
- "num_tokens": 685500.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
"step": 167,
- "step_time": 3.1906087530005607
+ "step_time": 11.708963983000103
},
{
"clip_ratio/high_max": 0.0,
@@ -4692,26 +4692,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 74.25,
- "completions/mean_terminated_length": 74.25,
- "completions/min_length": 16.0,
- "completions/min_terminated_length": 16.0,
- "entropy": 1.527149498462677,
- "epoch": 1.3658536585365852,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13164250552654266,
- "kl": 1.8541333702160046e-05,
- "learning_rate": 5.33641668598956e-07,
- "loss": -0.2347300797700882,
- "num_tokens": 688318.0,
- "reward": 0.7218117713928223,
- "reward_std": 0.1818692982196808,
- "rewards/true_env_reward_fn/mean": 0.7218117713928223,
- "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
"step": 168,
- "step_time": 8.50137474999974
+ "step_time": 16.905241970000134
},
{
"clip_ratio/high_max": 0.0,
@@ -4720,26 +4720,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 63.0,
- "completions/min_terminated_length": 63.0,
- "entropy": 1.3749513030052185,
- "epoch": 1.3739837398373984,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13246971368789673,
- "kl": 1.2620409506780561e-05,
- "learning_rate": 5.300681908752895e-07,
- "loss": 0.024534843862056732,
- "num_tokens": 692541.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
"step": 169,
- "step_time": 3.9512340759993094
+ "step_time": 12.27045077799994
},
{
"clip_ratio/high_max": 0.0,
@@ -4747,27 +4747,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 67.875,
- "completions/mean_terminated_length": 67.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2032299041748047,
- "epoch": 1.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11185794323682785,
- "kl": 1.5517784049734473e-05,
- "learning_rate": 5.264830667587295e-07,
- "loss": -0.05245225131511688,
- "num_tokens": 698064.0,
- "reward": 0.4444866180419922,
- "reward_std": 0.32400256395339966,
- "rewards/true_env_reward_fn/mean": 0.4444866180419922,
- "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
"step": 170,
- "step_time": 4.471538110999973
+ "step_time": 31.721865252000498
},
{
"clip_ratio/high_max": 0.0,
@@ -4776,26 +4776,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 95.0,
- "completions/mean_terminated_length": 95.0,
- "completions/min_length": 79.0,
- "completions/min_terminated_length": 79.0,
- "entropy": 1.4204387068748474,
- "epoch": 1.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1646534949541092,
- "kl": 2.4697198568901513e-05,
- "learning_rate": 5.228866172636899e-07,
- "loss": 0.02632315456867218,
- "num_tokens": 704196.0,
- "reward": 0.304565966129303,
- "reward_std": 0.32997164130210876,
- "rewards/true_env_reward_fn/mean": 0.304565966129303,
- "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
"step": 171,
- "step_time": 5.0436168590003945
+ "step_time": 8.973740739000277
},
{
"clip_ratio/high_max": 0.0,
@@ -4804,26 +4804,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1254178285598755,
- "epoch": 1.3983739837398375,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15058821439743042,
- "kl": 1.8407325114822015e-05,
- "learning_rate": 5.192791644186662e-07,
- "loss": 0.025478817522525787,
- "num_tokens": 706411.0,
- "reward": 0.7279239892959595,
- "reward_std": 0.11376125365495682,
- "rewards/true_env_reward_fn/mean": 0.7279239892959595,
- "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
"step": 172,
- "step_time": 3.2261944119982218
+ "step_time": 10.39117795600032
},
{
"clip_ratio/high_max": 0.0,
@@ -4832,26 +4832,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0894773602485657,
- "epoch": 1.4065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1687694787979126,
- "kl": 2.6046765015053097e-05,
- "learning_rate": 5.156610312374013e-07,
- "loss": -0.056941211223602295,
- "num_tokens": 711212.0,
- "reward": 0.4907146692276001,
- "reward_std": 0.3376546800136566,
- "rewards/true_env_reward_fn/mean": 0.4907146692276001,
- "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
"step": 173,
- "step_time": 3.9498180619993946
+ "step_time": 11.8111169230001
},
{
"clip_ratio/high_max": 0.0,
@@ -4860,26 +4860,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 60.125,
- "completions/mean_terminated_length": 60.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2238691449165344,
- "epoch": 1.4146341463414633,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19584441184997559,
- "kl": 4.8285241064149886e-05,
- "learning_rate": 5.120325416899629e-07,
- "loss": 0.0766875222325325,
- "num_tokens": 715409.0,
- "reward": 0.4593355059623718,
- "reward_std": 0.3909546136856079,
- "rewards/true_env_reward_fn/mean": 0.4593355059623718,
- "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
"step": 174,
- "step_time": 4.100519798999812
+ "step_time": 17.211099596999702
},
{
"clip_ratio/high_max": 0.0,
@@ -4888,26 +4888,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1763202548027039,
- "epoch": 1.4227642276422765,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1448555737733841,
- "kl": 1.2618989785551094e-05,
- "learning_rate": 5.08394020673734e-07,
- "loss": -0.012558378279209137,
- "num_tokens": 722327.0,
- "reward": 0.15966665744781494,
- "reward_std": 0.3235519230365753,
- "rewards/true_env_reward_fn/mean": 0.15966665744781494,
- "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
"step": 175,
- "step_time": 3.875348296000084
+ "step_time": 12.29185969800028
},
{
"clip_ratio/high_max": 0.0,
@@ -4916,26 +4916,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2852763533592224,
- "epoch": 1.4308943089430894,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22303813695907593,
- "kl": 4.580334098136518e-05,
- "learning_rate": 5.047457939843227e-07,
- "loss": -0.09214464575052261,
- "num_tokens": 726828.0,
- "reward": 0.25830498337745667,
- "reward_std": 0.37860655784606934,
- "rewards/true_env_reward_fn/mean": 0.25830498337745667,
- "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
"step": 176,
- "step_time": 4.241473076999682
+ "step_time": 10.987576109999736
},
{
"clip_ratio/high_max": 0.0,
@@ -4944,26 +4944,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.3609731197357178,
- "epoch": 1.4390243902439024,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.001985745271668e-05,
- "kl": 1.4942165307729738e-05,
- "learning_rate": 5.010881882863893e-07,
- "loss": 7.44550789022469e-07,
- "num_tokens": 729930.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
"step": 177,
- "step_time": 3.3902666960002534
+ "step_time": 10.792315139000493
},
{
"clip_ratio/high_max": 0.0,
@@ -4972,26 +4972,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2678966522216797,
- "epoch": 1.4471544715447155,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15351757407188416,
- "kl": 1.6737100395403104e-05,
- "learning_rate": 4.974215310843967e-07,
- "loss": 0.041131969541311264,
- "num_tokens": 732155.0,
- "reward": 0.7803820371627808,
- "reward_std": 0.08667682856321335,
- "rewards/true_env_reward_fn/mean": 0.7803820371627808,
- "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
"step": 178,
- "step_time": 3.637111981999624
+ "step_time": 12.399353334999887
},
{
"clip_ratio/high_max": 0.0,
@@ -5000,26 +5000,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.146271526813507,
- "epoch": 1.4552845528455285,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18466196954250336,
- "kl": 3.719841197380447e-05,
- "learning_rate": 4.937461506932859e-07,
- "loss": 0.029051154851913452,
- "num_tokens": 735418.0,
- "reward": 0.40377071499824524,
- "reward_std": 0.28345924615859985,
- "rewards/true_env_reward_fn/mean": 0.40377071499824524,
- "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
"step": 179,
- "step_time": 3.340555791999577
+ "step_time": 12.45691162600042
},
{
"clip_ratio/high_max": 0.0,
@@ -5028,26 +5028,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 170.0,
- "completions/max_terminated_length": 170.0,
- "completions/mean_length": 79.75,
- "completions/mean_terminated_length": 79.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.3902945518493652,
- "epoch": 1.4634146341463414,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1020251139998436,
- "kl": 1.8220500351162627e-05,
- "learning_rate": 4.900623762090777e-07,
- "loss": -0.002344265580177307,
- "num_tokens": 740540.0,
- "reward": 0.3590222895145416,
- "reward_std": 0.12487777322530746,
- "rewards/true_env_reward_fn/mean": 0.3590222895145416,
- "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
"step": 180,
- "step_time": 7.219923718001155
+ "step_time": 12.108760526000424
},
{
"clip_ratio/high_max": 0.0,
@@ -5056,26 +5056,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.517557680606842,
- "epoch": 1.4715447154471546,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24192294478416443,
- "kl": 4.1268089262302965e-05,
- "learning_rate": 4.863705374794055e-07,
- "loss": 0.09132950007915497,
- "num_tokens": 744723.0,
- "reward": 0.23991000652313232,
- "reward_std": 0.2837013602256775,
- "rewards/true_env_reward_fn/mean": 0.23991000652313232,
- "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
"step": 181,
- "step_time": 3.696339096999509
+ "step_time": 14.16400909000049
},
{
"clip_ratio/high_max": 0.0,
@@ -5084,26 +5084,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.249614655971527,
- "epoch": 1.4796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1566745787858963,
- "kl": 2.6629099920683075e-05,
- "learning_rate": 4.826709650739811e-07,
- "loss": 0.003972277045249939,
- "num_tokens": 748979.0,
- "reward": 0.4935140311717987,
- "reward_std": 0.41978561878204346,
- "rewards/true_env_reward_fn/mean": 0.4935140311717987,
- "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
"step": 182,
- "step_time": 3.316512920000605
+ "step_time": 19.767916835999586
},
{
"clip_ratio/high_max": 0.0,
@@ -5112,26 +5112,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1666916608810425,
- "epoch": 1.4878048780487805,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001005820304271765,
- "kl": 1.2246940059412736e-05,
- "learning_rate": 4.789639902549948e-07,
- "loss": 6.330609494398232e-07,
- "num_tokens": 751323.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
"step": 183,
- "step_time": 3.7099916660008603
+ "step_time": 10.428195530999346
},
{
"clip_ratio/high_max": 0.0,
@@ -5140,26 +5140,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.3246222138404846,
- "epoch": 1.4959349593495934,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18140996992588043,
- "kl": 3.1042441150930244e-05,
- "learning_rate": 4.752499449474535e-07,
- "loss": -0.022353097796440125,
- "num_tokens": 755494.0,
- "reward": 0.4561777412891388,
- "reward_std": 0.2439236342906952,
- "rewards/true_env_reward_fn/mean": 0.4561777412891388,
- "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
"step": 184,
- "step_time": 3.7916486710000754
+ "step_time": 16.47171987999991
},
{
"clip_ratio/high_max": 0.0,
@@ -5168,26 +5168,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 75.75,
- "completions/mean_terminated_length": 75.75,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.2625707983970642,
- "epoch": 1.5040650406504064,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11702944338321686,
- "kl": 1.503958355897339e-05,
- "learning_rate": 4.715291617094607e-07,
- "loss": 0.023916304111480713,
- "num_tokens": 758432.0,
- "reward": 0.5015827417373657,
- "reward_std": 0.1783808320760727,
- "rewards/true_env_reward_fn/mean": 0.5015827417373657,
- "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
"step": 185,
- "step_time": 4.226409274000616
+ "step_time": 13.684267182999974
},
{
"clip_ratio/high_max": 0.0,
@@ -5196,26 +5196,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 54.0,
- "completions/mean_terminated_length": 54.0,
- "completions/min_length": 25.0,
- "completions/min_terminated_length": 25.0,
- "entropy": 1.1460023522377014,
- "epoch": 1.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11851620674133301,
- "kl": 1.3728345948038623e-05,
- "learning_rate": 4.678019737024387e-07,
- "loss": 0.0831337422132492,
- "num_tokens": 764336.0,
- "reward": 0.19094166159629822,
- "reward_std": 0.30934420228004456,
- "rewards/true_env_reward_fn/mean": 0.19094166159629822,
- "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
"step": 186,
- "step_time": 3.563357556000483
+ "step_time": 10.281018189999031
},
{
"clip_ratio/high_max": 0.0,
@@ -5224,26 +5224,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1291148662567139,
- "epoch": 1.5203252032520327,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.864256960805506e-05,
- "kl": 1.081683785741916e-05,
- "learning_rate": 4.6406871466129704e-07,
- "loss": 5.412177870312007e-07,
- "num_tokens": 766608.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
"step": 187,
- "step_time": 3.534869859002356
+ "step_time": 24.351223329000277
},
{
"clip_ratio/high_max": 0.0,
@@ -5252,26 +5252,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 92.0,
- "completions/mean_terminated_length": 92.0,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3645328283309937,
- "epoch": 1.5284552845528454,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18168245255947113,
- "kl": 2.6857565899263136e-05,
- "learning_rate": 4.6032971886454956e-07,
- "loss": -0.005156125873327255,
- "num_tokens": 774496.0,
- "reward": 0.11249999701976776,
- "reward_std": 0.20856082439422607,
- "rewards/true_env_reward_fn/mean": 0.11249999701976776,
- "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
"step": 188,
- "step_time": 8.34005261099992
+ "step_time": 10.644911742000204
},
{
"clip_ratio/high_max": 0.0,
@@ -5280,26 +5280,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1495982110500336,
- "epoch": 1.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14486820995807648,
- "kl": 1.3202762147557223e-05,
- "learning_rate": 4.5658532110438337e-07,
- "loss": -0.0010610297322273254,
- "num_tokens": 777186.0,
- "reward": 0.5879127383232117,
- "reward_std": 0.05142820253968239,
- "rewards/true_env_reward_fn/mean": 0.5879127383232117,
- "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
"step": 189,
- "step_time": 3.4856022139993
+ "step_time": 9.579594637999435
},
{
"clip_ratio/high_max": 0.0,
@@ -5308,26 +5308,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.241140365600586,
- "epoch": 1.5447154471544715,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001250067143701017,
- "kl": 1.5482702110602986e-05,
- "learning_rate": 4.52835856656681e-07,
- "loss": 7.80837922320643e-07,
- "num_tokens": 779965.0,
- "reward": 0.6861198544502258,
- "reward_std": 0.08807206153869629,
- "rewards/true_env_reward_fn/mean": 0.6861198544502258,
- "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
"step": 190,
- "step_time": 3.904181735999373
+ "step_time": 11.495368679999501
},
{
"clip_ratio/high_max": 0.0,
@@ -5336,26 +5336,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2944807410240173,
- "epoch": 1.5528455284552845,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2030111849308014,
- "kl": 3.585523518268019e-05,
- "learning_rate": 4.490816612509991e-07,
- "loss": 0.0143373291939497,
- "num_tokens": 786140.0,
- "reward": 0.35173332691192627,
- "reward_std": 0.18115806579589844,
- "rewards/true_env_reward_fn/mean": 0.35173332691192627,
- "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
"step": 191,
- "step_time": 4.005758510000305
+ "step_time": 7.909104242000012
},
{
"clip_ratio/high_max": 0.0,
@@ -5364,26 +5364,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 58.0,
- "completions/mean_terminated_length": 58.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2177271246910095,
- "epoch": 1.5609756097560976,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11188288033008575,
- "kl": 2.3622495064046234e-05,
- "learning_rate": 4.45323071040508e-07,
- "loss": -0.02369789034128189,
- "num_tokens": 790424.0,
- "reward": 0.4888629913330078,
- "reward_std": 0.23310808837413788,
- "rewards/true_env_reward_fn/mean": 0.4888629913330078,
- "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
"step": 192,
- "step_time": 3.2518814809991454
+ "step_time": 10.043768619000275
},
{
"clip_ratio/high_max": 0.0,
@@ -5392,26 +5392,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.192966103553772,
- "epoch": 1.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1378823071718216,
- "kl": 1.7358055174554465e-05,
- "learning_rate": 4.4156042257189143e-07,
- "loss": 0.06256310641765594,
- "num_tokens": 794521.0,
- "reward": 0.5219699740409851,
- "reward_std": 0.06214587390422821,
- "rewards/true_env_reward_fn/mean": 0.5219699740409851,
- "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
"step": 193,
- "step_time": 4.222739491999164
+ "step_time": 10.19648474899941
},
{
"clip_ratio/high_max": 0.0,
@@ -5420,26 +5420,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1770159006118774,
- "epoch": 1.5772357723577235,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19550059735774994,
- "kl": 2.4871268578863237e-05,
- "learning_rate": 4.377940527552125e-07,
- "loss": 0.05841376632452011,
- "num_tokens": 798194.0,
- "reward": 0.42302167415618896,
- "reward_std": 0.2911272346973419,
- "rewards/true_env_reward_fn/mean": 0.42302167415618896,
- "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
"step": 194,
- "step_time": 3.8170270639984665
+ "step_time": 14.372816425000565
},
{
"clip_ratio/high_max": 0.0,
@@ -5448,26 +5448,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.5244255661964417,
- "epoch": 1.5853658536585367,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1872202306985855,
- "kl": 2.5990483663917985e-05,
- "learning_rate": 4.340242988337462e-07,
- "loss": -0.044112429022789,
- "num_tokens": 802802.0,
- "reward": 0.33745431900024414,
- "reward_std": 0.22955451905727386,
- "rewards/true_env_reward_fn/mean": 0.33745431900024414,
- "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
"step": 195,
- "step_time": 4.27381555500142
+ "step_time": 11.282298853000157
},
{
"clip_ratio/high_max": 0.0,
@@ -5476,26 +5476,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.345891296863556,
- "epoch": 1.5934959349593496,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16891054809093475,
- "kl": 2.7261638024356216e-05,
- "learning_rate": 4.3025149835378275e-07,
- "loss": -0.139386385679245,
- "num_tokens": 807881.0,
- "reward": 0.3240283131599426,
- "reward_std": 0.2803676128387451,
- "rewards/true_env_reward_fn/mean": 0.3240283131599426,
- "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
"step": 196,
- "step_time": 4.981287381999209
+ "step_time": 8.65745804199969
},
{
"clip_ratio/high_max": 0.0,
@@ -5504,26 +5504,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.246802031993866,
- "epoch": 1.6016260162601625,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2096729576587677,
- "kl": 3.5958016269432846e-05,
- "learning_rate": 4.2647598913440264e-07,
- "loss": -0.02941281348466873,
- "num_tokens": 812500.0,
- "reward": 0.4126526415348053,
- "reward_std": 0.36393746733665466,
- "rewards/true_env_reward_fn/mean": 0.4126526415348053,
- "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
"step": 197,
- "step_time": 3.101726017999681
+ "step_time": 9.615110594999805
},
{
"clip_ratio/high_max": 0.0,
@@ -5532,26 +5532,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4369062185287476,
- "epoch": 1.6097560975609757,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1862105429172516,
- "kl": 4.7646244638599455e-05,
- "learning_rate": 4.2269810923722965e-07,
- "loss": 0.0521145761013031,
- "num_tokens": 818718.0,
- "reward": 0.26869943737983704,
- "reward_std": 0.1483483463525772,
- "rewards/true_env_reward_fn/mean": 0.26869943737983704,
- "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
"step": 198,
- "step_time": 4.287780451000799
+ "step_time": 10.832162847999825
},
{
"clip_ratio/high_max": 0.0,
@@ -5560,26 +5560,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 21.0,
- "completions/min_terminated_length": 21.0,
- "entropy": 1.192937195301056,
- "epoch": 1.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2013663798570633,
- "kl": 1.8760739294521045e-05,
- "learning_rate": 4.189181969361588e-07,
- "loss": 0.07236722111701965,
- "num_tokens": 825728.0,
- "reward": 0.23110000789165497,
- "reward_std": 0.23212090134620667,
- "rewards/true_env_reward_fn/mean": 0.23110000789165497,
- "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
"step": 199,
- "step_time": 4.708717262998107
+ "step_time": 10.94759418800004
},
{
"clip_ratio/high_max": 0.0,
@@ -5588,26 +5588,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.3792839050292969,
- "epoch": 1.6260162601626016,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.7985117463395e-05,
- "kl": 1.4280476534622721e-05,
- "learning_rate": 4.1513659068706814e-07,
- "loss": 7.153485626076872e-07,
- "num_tokens": 830318.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
"step": 200,
- "step_time": 3.9055351140013954
+ "step_time": 10.322844021000492
},
{
"clip_ratio/high_max": 0.0,
@@ -5616,26 +5616,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.0505937337875366,
- "epoch": 1.6341463414634148,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13188514113426208,
- "kl": 2.842090088961413e-05,
- "learning_rate": 4.1135362909751326e-07,
- "loss": -0.0017508119344711304,
- "num_tokens": 834933.0,
- "reward": 0.36545002460479736,
- "reward_std": 0.24526984989643097,
- "rewards/true_env_reward_fn/mean": 0.36545002460479736,
- "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.083335876464844,
+ "completions/mean_terminated_length": 62.083335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.319727510213852,
+ "epoch": 4.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08774503320455551,
+ "kl": 6.86226876496221e-05,
+ "learning_rate": 6.077002463994907e-07,
+ "loss": 0.058894164860248566,
+ "num_tokens": 4956466.0,
+ "reward": 0.41384777426719666,
+ "reward_std": 0.17193447053432465,
+ "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+ "rewards/true_env_reward_fn/std": 0.17193445563316345,
"step": 201,
- "step_time": 3.895525625997834
+ "step_time": 10.146928047999609
},
{
"clip_ratio/high_max": 0.0,
@@ -5644,26 +5644,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 163.0,
- "completions/max_terminated_length": 163.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.3750707507133484,
- "epoch": 1.6422764227642277,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1798955649137497,
- "kl": 2.5428611479583196e-05,
- "learning_rate": 4.075696508964076e-07,
- "loss": 0.1843666434288025,
- "num_tokens": 838076.0,
- "reward": 0.44641831517219543,
- "reward_std": 0.30040720105171204,
- "rewards/true_env_reward_fn/mean": 0.44641831517219543,
- "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 59.395835876464844,
+ "completions/mean_terminated_length": 59.395835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1935442388057709,
+ "epoch": 4.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08328510820865631,
+ "kl": 7.948942857183283e-05,
+ "learning_rate": 6.035394200708103e-07,
+ "loss": 0.045964501798152924,
+ "num_tokens": 4978733.0,
+ "reward": 0.4896667003631592,
+ "reward_std": 0.3830767571926117,
+ "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+ "rewards/true_env_reward_fn/std": 0.3830767571926117,
"step": 202,
- "step_time": 6.74463491500137
+ "step_time": 10.025533761000133
},
{
"clip_ratio/high_max": 0.0,
@@ -5672,26 +5672,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 79.125,
- "completions/mean_terminated_length": 79.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3369249105453491,
- "epoch": 1.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11249764263629913,
- "kl": 1.1453126717242412e-05,
- "learning_rate": 4.0378499490369267e-07,
- "loss": -0.08210685849189758,
- "num_tokens": 841933.0,
- "reward": 0.3737962245941162,
- "reward_std": 0.13184049725532532,
- "rewards/true_env_reward_fn/mean": 0.3737962245941162,
- "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 62.22916793823242,
+ "completions/mean_terminated_length": 62.22916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2698631286621094,
+ "epoch": 4.951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07158155739307404,
+ "kl": 8.286665615742095e-05,
+ "learning_rate": 5.993710887521302e-07,
+ "loss": -0.03551984950900078,
+ "num_tokens": 5009440.0,
+ "reward": 0.45639634132385254,
+ "reward_std": 0.1837630569934845,
+ "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+ "rewards/true_env_reward_fn/std": 0.1837630420923233,
"step": 203,
- "step_time": 5.5639925510004105
+ "step_time": 9.823523802999716
},
{
"clip_ratio/high_max": 0.0,
@@ -5700,26 +5700,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.5737199783325195,
- "epoch": 1.6585365853658538,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19394491612911224,
- "kl": 5.05317857459886e-05,
- "learning_rate": 4e-07,
- "loss": 0.05622926354408264,
- "num_tokens": 847711.0,
- "reward": 0.10725000500679016,
- "reward_std": 0.19980257749557495,
- "rewards/true_env_reward_fn/mean": 0.10725000500679016,
- "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "entropy": 1.2825455367565155,
+ "epoch": 4.975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07431793212890625,
+ "kl": 7.247529447340639e-05,
+ "learning_rate": 5.951955545823342e-07,
+ "loss": -0.02418600022792816,
+ "num_tokens": 5036036.0,
+ "reward": 0.4476773142814636,
+ "reward_std": 0.20447416603565216,
+ "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+ "rewards/true_env_reward_fn/std": 0.20447418093681335,
"step": 204,
- "step_time": 4.918089437000162
+ "step_time": 11.960790695000014
},
{
"clip_ratio/high_max": 0.0,
@@ -5728,26 +5728,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 60.5,
- "completions/mean_terminated_length": 60.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.119917094707489,
- "epoch": 1.6666666666666665,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10553985089063644,
- "kl": 1.2793303994840244e-05,
- "learning_rate": 3.9621500509630725e-07,
- "loss": -0.011355768889188766,
- "num_tokens": 849519.0,
- "reward": 0.6593211889266968,
- "reward_std": 0.11862105131149292,
- "rewards/true_env_reward_fn/mean": 0.6593211889266968,
- "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2438389956951141,
+ "epoch": 5.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06833480298519135,
+ "kl": 5.646793260893901e-05,
+ "learning_rate": 5.91013120222401e-07,
+ "loss": 0.0021926667541265488,
+ "num_tokens": 5063228.0,
+ "reward": 0.4300723075866699,
+ "reward_std": 0.12116922438144684,
+ "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+ "rewards/true_env_reward_fn/std": 0.12116922438144684,
"step": 205,
- "step_time": 4.127652793999005
+ "step_time": 9.874485716000436
},
{
"clip_ratio/high_max": 0.0,
@@ -5756,26 +5756,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 75.125,
- "completions/mean_terminated_length": 75.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3856809735298157,
- "epoch": 1.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1911258101463318,
- "kl": 4.2569914512569085e-05,
- "learning_rate": 3.9243034910359247e-07,
- "loss": 0.10561336576938629,
- "num_tokens": 854156.0,
- "reward": 0.2516202926635742,
- "reward_std": 0.22357939183712006,
- "rewards/true_env_reward_fn/mean": 0.2516202926635742,
- "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "completions/max_length": 205.0,
+ "completions/max_terminated_length": 205.0,
+ "completions/mean_length": 66.1875,
+ "completions/mean_terminated_length": 66.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2645181119441986,
+ "epoch": 5.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08248262107372284,
+ "kl": 9.333990783488844e-05,
+ "learning_rate": 5.868240888334652e-07,
+ "loss": 0.12253005057573318,
+ "num_tokens": 5093373.0,
+ "reward": 0.28542038798332214,
+ "reward_std": 0.326623797416687,
+ "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+ "rewards/true_env_reward_fn/std": 0.326623797416687,
"step": 206,
- "step_time": 4.714620994000143
+ "step_time": 15.706792760000553
},
{
"clip_ratio/high_max": 0.0,
@@ -5784,26 +5784,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.4012945890426636,
- "epoch": 1.6829268292682928,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226125180721283,
- "kl": 5.4918069963605376e-05,
- "learning_rate": 3.886463709024868e-07,
- "loss": 0.012949362397193909,
- "num_tokens": 856151.0,
- "reward": 0.6204804182052612,
- "reward_std": 0.44673967361450195,
- "rewards/true_env_reward_fn/mean": 0.6204804182052612,
- "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.16667175292969,
+ "completions/mean_terminated_length": 67.16667175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.34083291888237,
+ "epoch": 5.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07931552827358246,
+ "kl": 0.00010715152166085318,
+ "learning_rate": 5.826287640548424e-07,
+ "loss": 0.0005592256784439087,
+ "num_tokens": 5118933.0,
+ "reward": 0.5035215616226196,
+ "reward_std": 0.304157018661499,
+ "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+ "rewards/true_env_reward_fn/std": 0.30415698885917664,
"step": 207,
- "step_time": 4.094810713999323
+ "step_time": 10.04168460900064
},
{
"clip_ratio/high_max": 0.0,
@@ -5812,26 +5812,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3708943128585815,
- "epoch": 1.6910569105691056,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12903675436973572,
- "kl": 1.4145812656352064e-05,
- "learning_rate": 3.8486340931293187e-07,
- "loss": -0.07838249206542969,
- "num_tokens": 859380.0,
- "reward": 0.6400156021118164,
- "reward_std": 0.1022576317191124,
- "rewards/true_env_reward_fn/mean": 0.6400156021118164,
- "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 73.4375,
+ "completions/mean_terminated_length": 73.4375,
+ "completions/min_length": 11.0,
+ "completions/min_terminated_length": 11.0,
+ "entropy": 1.1821868121623993,
+ "epoch": 5.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07619761675596237,
+ "kl": 0.00013189401397539768,
+ "learning_rate": 5.784274499820213e-07,
+ "loss": -0.011967688798904419,
+ "num_tokens": 5148682.0,
+ "reward": 0.36097532510757446,
+ "reward_std": 0.29208436608314514,
+ "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+ "rewards/true_env_reward_fn/std": 0.29208436608314514,
"step": 208,
- "step_time": 4.297900428997309
+ "step_time": 15.226898961000188
},
{
"clip_ratio/high_max": 0.0,
@@ -5840,26 +5840,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.75,
- "completions/mean_terminated_length": 56.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2782961130142212,
- "epoch": 1.6991869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12437833100557327,
- "kl": 1.4473939245362999e-05,
- "learning_rate": 3.8108180306384135e-07,
- "loss": -0.036324724555015564,
- "num_tokens": 862714.0,
- "reward": 0.43419933319091797,
- "reward_std": 0.15345513820648193,
- "rewards/true_env_reward_fn/mean": 0.43419933319091797,
- "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 75.39583587646484,
+ "completions/mean_terminated_length": 75.39583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2636725008487701,
+ "epoch": 5.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06586338579654694,
+ "kl": 1.9426477138040354e-05,
+ "learning_rate": 5.742204511446203e-07,
+ "loss": -0.047095417976379395,
+ "num_tokens": 5173205.0,
+ "reward": 0.5261925458908081,
+ "reward_std": 0.2689943313598633,
+ "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+ "rewards/true_env_reward_fn/std": 0.2689943313598633,
"step": 209,
- "step_time": 3.47861851900052
+ "step_time": 23.690397457000017
},
{
"clip_ratio/high_max": 0.0,
@@ -5867,27 +5867,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 123.875,
- "completions/mean_terminated_length": 68.42857360839844,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.5170292258262634,
- "epoch": 1.7073170731707317,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 54.3125,
+ "completions/mean_terminated_length": 54.3125,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2415392696857452,
+ "epoch": 5.121951219512195,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1543467938899994,
- "kl": 2.3121557205740828e-05,
- "learning_rate": 3.7730189076277037e-07,
- "loss": -0.3810324966907501,
- "num_tokens": 869789.0,
- "reward": 0.3795333504676819,
- "reward_std": 0.13815106451511383,
- "rewards/true_env_reward_fn/mean": 0.3795333504676819,
- "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "grad_norm": 0.0990133062005043,
+ "kl": 7.119746442185715e-05,
+ "learning_rate": 5.700080724843146e-07,
+ "loss": -0.030588299036026,
+ "num_tokens": 5199892.0,
+ "reward": 0.3418487012386322,
+ "reward_std": 0.36353805661201477,
+ "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+ "rewards/true_env_reward_fn/std": 0.3635380268096924,
"step": 210,
- "step_time": 20.698896928999602
+ "step_time": 9.864614251999683
},
{
"clip_ratio/high_max": 0.0,
@@ -5896,26 +5896,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2324861884117126,
- "epoch": 1.7154471544715446,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11762262135744095,
- "kl": 1.9743249595194357e-05,
- "learning_rate": 3.735240108655973e-07,
- "loss": -0.09104303270578384,
- "num_tokens": 872604.0,
- "reward": 0.5916227102279663,
- "reward_std": 0.1752101182937622,
- "rewards/true_env_reward_fn/mean": 0.5916227102279663,
- "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2508135735988617,
+ "epoch": 5.146341463414634,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10835102945566177,
+ "kl": 0.00012395972225931473,
+ "learning_rate": 5.657906193327324e-07,
+ "loss": 0.08621911704540253,
+ "num_tokens": 5220141.0,
+ "reward": 0.4987506866455078,
+ "reward_std": 0.27843451499938965,
+ "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+ "rewards/true_env_reward_fn/std": 0.27843451499938965,
"step": 211,
- "step_time": 5.218213289999767
+ "step_time": 9.983622502000344
},
{
"clip_ratio/high_max": 0.0,
@@ -5924,26 +5924,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.4448966979980469,
- "epoch": 1.7235772357723578,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1924143135547638,
- "kl": 3.186432604707079e-05,
- "learning_rate": 3.697485016462174e-07,
- "loss": 0.025449808686971664,
- "num_tokens": 877921.0,
- "reward": 0.1028124988079071,
- "reward_std": 0.2254277467727661,
- "rewards/true_env_reward_fn/mean": 0.1028124988079071,
- "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1611086130142212,
+ "epoch": 5.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08423946797847748,
+ "kl": 4.4347343191475375e-05,
+ "learning_rate": 5.615683973893234e-07,
+ "loss": 0.08670675754547119,
+ "num_tokens": 5244470.0,
+ "reward": 0.4252437949180603,
+ "reward_std": 0.2996494472026825,
+ "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+ "rewards/true_env_reward_fn/std": 0.2996494472026825,
"step": 212,
- "step_time": 3.415528882000217
+ "step_time": 18.68646409699977
},
{
"clip_ratio/high_max": 0.0,
@@ -5952,26 +5952,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
"completions/min_length": 32.0,
"completions/min_terminated_length": 32.0,
- "entropy": 1.185910701751709,
- "epoch": 1.7317073170731707,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001214127623825334,
- "kl": 1.3199866316426778e-05,
- "learning_rate": 3.659757011662538e-07,
- "loss": 6.776077725589857e-07,
- "num_tokens": 880344.0,
- "reward": 0.7329437732696533,
- "reward_std": 0.22123214602470398,
- "rewards/true_env_reward_fn/mean": 0.7329437732696533,
- "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "entropy": 1.3110275566577911,
+ "epoch": 5.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06817487627267838,
+ "kl": 0.00010393545744591393,
+ "learning_rate": 5.573417126992002e-07,
+ "loss": 0.04062439873814583,
+ "num_tokens": 5271369.0,
+ "reward": 0.4340522885322571,
+ "reward_std": 0.26940545439720154,
+ "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+ "rewards/true_env_reward_fn/std": 0.26940542459487915,
"step": 213,
- "step_time": 3.5156538789997285
+ "step_time": 11.3590317649996
},
{
"clip_ratio/high_max": 0.0,
@@ -5980,26 +5980,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 162.0,
- "completions/max_terminated_length": 162.0,
- "completions/mean_length": 90.75,
- "completions/mean_terminated_length": 90.75,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2697569727897644,
- "epoch": 1.7398373983739837,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 54.4375,
+ "completions/mean_terminated_length": 54.4375,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2018881738185883,
+ "epoch": 5.219512195121951,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1553589552640915,
- "kl": 1.621047795197228e-05,
- "learning_rate": 3.622059472447875e-07,
- "loss": -0.111361563205719,
- "num_tokens": 885006.0,
- "reward": 0.5490846633911133,
- "reward_std": 0.14710450172424316,
- "rewards/true_env_reward_fn/mean": 0.5490846633911133,
- "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "grad_norm": 0.10059589147567749,
+ "kl": 0.00011373830420779996,
+ "learning_rate": 5.531108716309547e-07,
+ "loss": -0.09816272556781769,
+ "num_tokens": 5288638.0,
+ "reward": 0.5691710710525513,
+ "reward_std": 0.25253745913505554,
+ "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+ "rewards/true_env_reward_fn/std": 0.25253745913505554,
"step": 214,
- "step_time": 7.051423932000034
+ "step_time": 9.633293675000004
},
{
"clip_ratio/high_max": 0.0,
@@ -6008,26 +6008,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1904898285865784,
- "epoch": 1.7479674796747968,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19009968638420105,
- "kl": 1.6167180092452327e-05,
- "learning_rate": 3.5843957742810864e-07,
- "loss": -0.006048411130905151,
- "num_tokens": 888255.0,
- "reward": 0.5893601179122925,
- "reward_std": 0.16021940112113953,
- "rewards/true_env_reward_fn/mean": 0.5893601179122925,
- "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1958912312984467,
+ "epoch": 5.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0779944583773613,
+ "kl": 7.025236300250981e-05,
+ "learning_rate": 5.488761808544509e-07,
+ "loss": 0.03371567651629448,
+ "num_tokens": 5312081.0,
+ "reward": 0.44519662857055664,
+ "reward_std": 0.22201679646968842,
+ "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+ "rewards/true_env_reward_fn/std": 0.22201678156852722,
"step": 215,
- "step_time": 3.0270869319992926
+ "step_time": 12.463993090999338
},
{
"clip_ratio/high_max": 0.0,
@@ -6036,26 +6036,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2750649452209473,
- "epoch": 1.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1521027386188507,
- "kl": 4.317680577514693e-05,
- "learning_rate": 3.5467692895949205e-07,
- "loss": -0.04247616231441498,
- "num_tokens": 892490.0,
- "reward": 0.1841849982738495,
- "reward_std": 0.30015870928764343,
- "rewards/true_env_reward_fn/mean": 0.1841849982738495,
- "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.2146256864070892,
+ "epoch": 5.2682926829268295,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06834157556295395,
+ "kl": 0.00010125362859980669,
+ "learning_rate": 5.446379473185971e-07,
+ "loss": -0.02198379673063755,
+ "num_tokens": 5334238.0,
+ "reward": 0.5273472666740417,
+ "reward_std": 0.21954773366451263,
+ "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+ "rewards/true_env_reward_fn/std": 0.21954771876335144,
"step": 216,
- "step_time": 3.461749838998003
+ "step_time": 15.126136884000061
},
{
"clip_ratio/high_max": 0.0,
@@ -6064,26 +6064,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2730335593223572,
- "epoch": 1.7642276422764227,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11831019073724747,
- "kl": 2.1095927877468057e-05,
- "learning_rate": 3.509183387490009e-07,
- "loss": -0.051231447607278824,
- "num_tokens": 894688.0,
- "reward": 0.49520131945610046,
- "reward_std": 0.3804744482040405,
- "rewards/true_env_reward_fn/mean": 0.49520131945610046,
- "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 61.333335876464844,
+ "completions/mean_terminated_length": 61.333335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2872387170791626,
+ "epoch": 5.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09578828513622284,
+ "kl": 7.404103780572768e-05,
+ "learning_rate": 5.403964782290961e-07,
+ "loss": -0.041091397404670715,
+ "num_tokens": 5351798.0,
+ "reward": 0.5464547872543335,
+ "reward_std": 0.16224616765975952,
+ "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+ "rewards/true_env_reward_fn/std": 0.16224615275859833,
"step": 217,
- "step_time": 3.7184635590019752
+ "step_time": 7.2901647220001
},
{
"clip_ratio/high_max": 0.0,
@@ -6092,26 +6092,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 47.75,
- "completions/mean_terminated_length": 47.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3015827536582947,
- "epoch": 1.7723577235772359,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21724702417850494,
- "kl": 3.562447545846226e-05,
- "learning_rate": 3.471641433433191e-07,
- "loss": -0.028775859624147415,
- "num_tokens": 899066.0,
- "reward": 0.3376166820526123,
- "reward_std": 0.21976198256015778,
- "rewards/true_env_reward_fn/mean": 0.3376166820526123,
- "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 84.89583587646484,
+ "completions/mean_terminated_length": 84.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.4322342276573181,
+ "epoch": 5.317073170731708,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0667513981461525,
+ "kl": 2.497344303264981e-05,
+ "learning_rate": 5.361520810261778e-07,
+ "loss": -0.06392769515514374,
+ "num_tokens": 5375369.0,
+ "reward": 0.5213420391082764,
+ "reward_std": 0.1436246931552887,
+ "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+ "rewards/true_env_reward_fn/std": 0.1436246782541275,
"step": 218,
- "step_time": 3.344433074000335
+ "step_time": 12.217936152999755
},
{
"clip_ratio/high_max": 0.0,
@@ -6120,26 +6120,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1390373706817627,
- "epoch": 1.7804878048780488,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001077545020962134,
- "kl": 1.554161144667887e-05,
- "learning_rate": 3.434146788956166e-07,
- "loss": 7.676237032683275e-07,
- "num_tokens": 902263.0,
- "reward": 0.5682899951934814,
- "reward_std": 0.1217179074883461,
- "rewards/true_env_reward_fn/mean": 0.5682899951934814,
- "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2382279634475708,
+ "epoch": 5.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07372384518384933,
+ "kl": 9.740726909512887e-05,
+ "learning_rate": 5.319050633623141e-07,
+ "loss": 0.010912742465734482,
+ "num_tokens": 5397956.0,
+ "reward": 0.5623860359191895,
+ "reward_std": 0.285375714302063,
+ "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+ "rewards/true_env_reward_fn/std": 0.285375714302063,
"step": 219,
- "step_time": 3.0424018219982827
+ "step_time": 13.449634822999997
},
{
"clip_ratio/high_max": 0.0,
@@ -6148,26 +6148,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1991714239120483,
- "epoch": 1.7886178861788617,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11428863555192947,
- "kl": 1.1265870853094384e-05,
- "learning_rate": 3.3967028113545045e-07,
- "loss": -0.02927359938621521,
- "num_tokens": 906206.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 65.39583587646484,
+ "completions/mean_terminated_length": 65.39583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2669419348239899,
+ "epoch": 5.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07886125147342682,
+ "kl": 0.00011165817522851285,
+ "learning_rate": 5.276557330799203e-07,
+ "loss": -0.02433733269572258,
+ "num_tokens": 5418511.0,
+ "reward": 0.6075118780136108,
+ "reward_std": 0.30748677253723145,
+ "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+ "rewards/true_env_reward_fn/std": 0.30748677253723145,
"step": 220,
- "step_time": 3.5531271640011255
+ "step_time": 12.051496982000117
},
{
"clip_ratio/high_max": 0.0,
@@ -6176,26 +6176,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2546668648719788,
- "epoch": 1.796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1504630595445633,
- "kl": 1.5663241811125772e-05,
- "learning_rate": 3.3593128533870314e-07,
- "loss": 0.04806854575872421,
- "num_tokens": 909149.0,
- "reward": 0.49599751830101013,
- "reward_std": 0.15429075062274933,
- "rewards/true_env_reward_fn/mean": 0.49599751830101013,
- "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2560602128505707,
+ "epoch": 5.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08607357740402222,
+ "kl": 8.424731549894204e-05,
+ "learning_rate": 5.234043981890393e-07,
+ "loss": 0.013634156435728073,
+ "num_tokens": 5439690.0,
+ "reward": 0.5080039501190186,
+ "reward_std": 0.21975299715995789,
+ "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+ "rewards/true_env_reward_fn/std": 0.21975299715995789,
"step": 221,
- "step_time": 3.1345955030010373
+ "step_time": 10.435893627000496
},
{
"clip_ratio/high_max": 0.0,
@@ -6204,26 +6204,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1428714394569397,
- "epoch": 1.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.310600969707593e-05,
- "kl": 1.1706195891747484e-05,
- "learning_rate": 3.321980262975613e-07,
- "loss": 5.957842290627013e-07,
- "num_tokens": 914211.0,
- "reward": 0.3258306384086609,
- "reward_std": 0.4338511824607849,
- "rewards/true_env_reward_fn/mean": 0.3258306384086609,
- "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3081265985965729,
+ "epoch": 5.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0876006931066513,
+ "kl": 9.127605972025776e-05,
+ "learning_rate": 5.191513668450177e-07,
+ "loss": -0.01968565583229065,
+ "num_tokens": 5468216.0,
+ "reward": 0.3679848611354828,
+ "reward_std": 0.237859845161438,
+ "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+ "rewards/true_env_reward_fn/std": 0.237859845161438,
"step": 222,
- "step_time": 3.8445859539988305
+ "step_time": 11.524256381999294
},
{
"clip_ratio/high_max": 0.0,
@@ -6232,26 +6232,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1629019975662231,
- "epoch": 1.8130081300813008,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010947151895379648,
- "kl": 1.3530024261854123e-05,
- "learning_rate": 3.2847083829053923e-07,
- "loss": 6.723923888785066e-07,
- "num_tokens": 916850.0,
- "reward": 0.5956059694290161,
- "reward_std": 0.09251586347818375,
- "rewards/true_env_reward_fn/mean": 0.5956059694290161,
- "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 67.66667175292969,
+ "completions/mean_terminated_length": 67.66667175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2990808486938477,
+ "epoch": 5.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07685395330190659,
+ "kl": 0.00014209141045284923,
+ "learning_rate": 5.148969473261679e-07,
+ "loss": -0.01107364147901535,
+ "num_tokens": 5488040.0,
+ "reward": 0.5435788035392761,
+ "reward_std": 0.35419100522994995,
+ "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+ "rewards/true_env_reward_fn/std": 0.35419100522994995,
"step": 223,
- "step_time": 3.3021794950000185
+ "step_time": 11.945272217000365
},
{
"clip_ratio/high_max": 0.0,
@@ -6260,26 +6260,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1237311363220215,
- "epoch": 1.821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14958374202251434,
- "kl": 4.083753810846247e-05,
- "learning_rate": 3.2475005505254657e-07,
- "loss": 0.09789139032363892,
- "num_tokens": 919901.0,
- "reward": 0.4179220199584961,
- "reward_std": 0.2486819326877594,
- "rewards/true_env_reward_fn/mean": 0.4179220199584961,
- "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.284348964691162,
+ "epoch": 5.463414634146342,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.058678388595581055,
+ "kl": 3.3754420201148605e-05,
+ "learning_rate": 5.106414480114238e-07,
+ "loss": -0.01714605651795864,
+ "num_tokens": 5513005.0,
+ "reward": 0.43836766481399536,
+ "reward_std": 0.26744046807289124,
+ "rewards/true_env_reward_fn/mean": 0.438367635011673,
+ "rewards/true_env_reward_fn/std": 0.26744046807289124,
"step": 224,
- "step_time": 4.954825423001239
+ "step_time": 10.563708176000091
},
{
"clip_ratio/high_max": 0.0,
@@ -6288,26 +6288,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 52.375,
- "completions/mean_terminated_length": 52.375,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1394256949424744,
- "epoch": 1.8292682926829267,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1480269879102707,
- "kl": 3.2915593692450784e-05,
- "learning_rate": 3.210360097450052e-07,
- "loss": 0.11319300532341003,
- "num_tokens": 923840.0,
- "reward": 0.4335233271121979,
- "reward_std": 0.31114333868026733,
- "rewards/true_env_reward_fn/mean": 0.4335233271121979,
- "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3357127904891968,
+ "epoch": 5.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07269197702407837,
+ "kl": 3.727909597728285e-05,
+ "learning_rate": 5.063851773579869e-07,
+ "loss": -0.01076439768075943,
+ "num_tokens": 5530655.0,
+ "reward": 0.6185358762741089,
+ "reward_std": 0.19721543788909912,
+ "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+ "rewards/true_env_reward_fn/std": 0.19721543788909912,
"step": 225,
- "step_time": 3.777259659000265
+ "step_time": 10.372150705000422
},
{
"clip_ratio/high_max": 0.0,
@@ -6316,26 +6316,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.354669451713562,
- "epoch": 1.8373983739837398,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13672730326652527,
- "kl": 2.2323275516100693e-05,
- "learning_rate": 3.173290349260188e-07,
- "loss": -0.0521523654460907,
- "num_tokens": 927882.0,
- "reward": 0.57341468334198,
- "reward_std": 0.11576741933822632,
- "rewards/true_env_reward_fn/mean": 0.57341468334198,
- "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 64.35417175292969,
+ "completions/mean_terminated_length": 64.35417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2975924611091614,
+ "epoch": 5.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0875314474105835,
+ "kl": 9.186910938296933e-05,
+ "learning_rate": 5.021284438789693e-07,
+ "loss": 0.048179637640714645,
+ "num_tokens": 5554032.0,
+ "reward": 0.45552024245262146,
+ "reward_std": 0.31553781032562256,
+ "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+ "rewards/true_env_reward_fn/std": 0.31553778052330017,
"step": 226,
- "step_time": 3.613498073998926
+ "step_time": 12.18860100899974
},
{
"clip_ratio/high_max": 0.0,
@@ -6344,26 +6344,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1750767230987549,
- "epoch": 1.845528455284553,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.230674147605896,
- "kl": 1.800864629331045e-05,
- "learning_rate": 3.136294625205945e-07,
- "loss": 0.058730173856019974,
- "num_tokens": 930225.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 63.208335876464844,
+ "completions/mean_terminated_length": 63.208335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2268341779708862,
+ "epoch": 5.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08318481594324112,
+ "kl": 8.052505017985823e-05,
+ "learning_rate": 4.978715561210307e-07,
+ "loss": 0.027963606640696526,
+ "num_tokens": 5582898.0,
+ "reward": 0.36310288310050964,
+ "reward_std": 0.3131144344806671,
+ "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+ "rewards/true_env_reward_fn/std": 0.3131144344806671,
"step": 227,
- "step_time": 3.3819083769976714
+ "step_time": 12.510411257000214
},
{
"clip_ratio/high_max": 0.0,
@@ -6372,26 +6372,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 40.875,
- "completions/mean_terminated_length": 40.875,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.2825847864151,
- "epoch": 1.8536585365853657,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.30391862988471985,
- "kl": 2.0106223928451072e-05,
- "learning_rate": 3.0993762379092235e-07,
- "loss": -0.10232458263635635,
- "num_tokens": 933048.0,
- "reward": 0.45667415857315063,
- "reward_std": 0.26290765404701233,
- "rewards/true_env_reward_fn/mean": 0.45667415857315063,
- "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 64.9375,
+ "completions/mean_terminated_length": 64.9375,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1668500006198883,
+ "epoch": 5.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0727877989411354,
+ "kl": 7.824771091691218e-05,
+ "learning_rate": 4.936148226420132e-07,
+ "loss": -0.01629078947007656,
+ "num_tokens": 5604791.0,
+ "reward": 0.5653349161148071,
+ "reward_std": 0.2849632799625397,
+ "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+ "rewards/true_env_reward_fn/std": 0.2849632799625397,
"step": 228,
- "step_time": 2.698590726000475
+ "step_time": 10.083805716999905
},
{
"clip_ratio/high_max": 0.0,
@@ -6400,26 +6400,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 41.125,
- "completions/mean_terminated_length": 41.125,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.3104369640350342,
- "epoch": 1.8617886178861789,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.28224605321884155,
- "kl": 6.227439189387951e-05,
- "learning_rate": 3.06253849306714e-07,
- "loss": -0.008679002523422241,
- "num_tokens": 938217.0,
- "reward": 0.2567799985408783,
- "reward_std": 0.2555168867111206,
- "rewards/true_env_reward_fn/mean": 0.2567799985408783,
- "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 68.29167175292969,
+ "completions/mean_terminated_length": 68.29167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.307439923286438,
+ "epoch": 5.585365853658536,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06797961890697479,
+ "kl": 6.601300174224889e-05,
+ "learning_rate": 4.893585519885763e-07,
+ "loss": -0.036383360624313354,
+ "num_tokens": 5631629.0,
+ "reward": 0.49251794815063477,
+ "reward_std": 0.22127023339271545,
+ "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+ "rewards/true_env_reward_fn/std": 0.22127023339271545,
"step": 229,
- "step_time": 2.7451177700022527
+ "step_time": 13.650024606999523
},
{
"clip_ratio/high_max": 0.0,
@@ -6428,26 +6428,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0164751410484314,
- "epoch": 1.8699186991869918,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011273886048002169,
- "kl": 1.1985231139988173e-05,
- "learning_rate": 3.0257846891560323e-07,
- "loss": 5.987301960885816e-07,
- "num_tokens": 941049.0,
- "reward": 0.6971603631973267,
- "reward_std": 0.2594861686229706,
- "rewards/true_env_reward_fn/mean": 0.6971603631973267,
- "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3524385392665863,
+ "epoch": 5.609756097560975,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.12484809756278992,
+ "kl": 0.0002558713749749586,
+ "learning_rate": 4.85103052673832e-07,
+ "loss": -0.0045075975358486176,
+ "num_tokens": 5666712.0,
+ "reward": 0.2605232000350952,
+ "reward_std": 0.22926004230976105,
+ "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+ "rewards/true_env_reward_fn/std": 0.22926005721092224,
"step": 230,
- "step_time": 2.917641182999432
+ "step_time": 15.322059910999542
},
{
"clip_ratio/high_max": 0.0,
@@ -6456,26 +6456,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 13.0,
- "completions/min_terminated_length": 13.0,
- "entropy": 1.1991845965385437,
- "epoch": 1.8780487804878048,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015363915008492768,
- "kl": 1.4284144981502322e-05,
- "learning_rate": 2.989118117136107e-07,
- "loss": 6.798551339670666e-07,
- "num_tokens": 945403.0,
- "reward": 0.31745320558547974,
- "reward_std": 0.42489534616470337,
- "rewards/true_env_reward_fn/mean": 0.31745320558547974,
- "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2202666401863098,
+ "epoch": 5.634146341463414,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07976125925779343,
+ "kl": 8.684267686476232e-05,
+ "learning_rate": 4.808486331549823e-07,
+ "loss": 0.012383833527565002,
+ "num_tokens": 5702004.0,
+ "reward": 0.20571085810661316,
+ "reward_std": 0.2265808880329132,
+ "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+ "rewards/true_env_reward_fn/std": 0.2265808880329132,
"step": 231,
- "step_time": 5.5585464220002905
+ "step_time": 14.180213787999492
},
{
"clip_ratio/high_max": 0.0,
@@ -6484,26 +6484,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2020843029022217,
- "epoch": 1.886178861788618,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13684362173080444,
- "kl": 3.651866154541494e-05,
- "learning_rate": 2.952542060156773e-07,
- "loss": 0.025869816541671753,
- "num_tokens": 949126.0,
- "reward": 0.3984018564224243,
- "reward_std": 0.4202974736690521,
- "rewards/true_env_reward_fn/mean": 0.3984018564224243,
- "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "completions/max_length": 312.0,
+ "completions/max_terminated_length": 312.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3096380531787872,
+ "epoch": 5.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08384906500577927,
+ "kl": 8.733692084206268e-05,
+ "learning_rate": 4.7659560181096067e-07,
+ "loss": 0.02098608762025833,
+ "num_tokens": 5729780.0,
+ "reward": 0.4599723219871521,
+ "reward_std": 0.21680118143558502,
+ "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+ "rewards/true_env_reward_fn/std": 0.21680118143558502,
"step": 232,
- "step_time": 3.4383463869999105
+ "step_time": 21.68401394800003
},
{
"clip_ratio/high_max": 0.0,
@@ -6512,26 +6512,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 58.625,
- "completions/mean_terminated_length": 58.625,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2224581837654114,
- "epoch": 1.8943089430894309,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20249564945697784,
- "kl": 3.106597978330683e-05,
- "learning_rate": 2.9160597932626605e-07,
- "loss": -0.003915777429938316,
- "num_tokens": 952815.0,
- "reward": 0.4141089916229248,
- "reward_std": 0.3064958453178406,
- "rewards/true_env_reward_fn/mean": 0.4141089916229248,
- "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 53.520835876464844,
+ "completions/mean_terminated_length": 53.520835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2158988416194916,
+ "epoch": 5.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08225176483392715,
+ "kl": 9.274652438762132e-05,
+ "learning_rate": 4.7234426692007977e-07,
+ "loss": -0.034079790115356445,
+ "num_tokens": 5753397.0,
+ "reward": 0.5273070335388184,
+ "reward_std": 0.2194610834121704,
+ "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+ "rewards/true_env_reward_fn/std": 0.21946106851100922,
"step": 233,
- "step_time": 3.6410487339999236
+ "step_time": 7.915307780000603
},
{
"clip_ratio/high_max": 0.0,
@@ -6540,26 +6540,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1212781071662903,
- "epoch": 1.9024390243902438,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14502985775470734,
- "kl": 3.0683338081871625e-05,
- "learning_rate": 2.879674583100372e-07,
- "loss": -0.0860406681895256,
- "num_tokens": 958656.0,
- "reward": 0.30326664447784424,
- "reward_std": 0.24800051748752594,
- "rewards/true_env_reward_fn/mean": 0.30326664447784424,
- "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 78.0,
+ "completions/mean_terminated_length": 78.0,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2362721860408783,
+ "epoch": 5.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06290840357542038,
+ "kl": 7.484563047910342e-05,
+ "learning_rate": 4.6809493663768575e-07,
+ "loss": -0.008873865008354187,
+ "num_tokens": 5778869.0,
+ "reward": 0.5283026695251465,
+ "reward_std": 0.195759579539299,
+ "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+ "rewards/true_env_reward_fn/std": 0.195759579539299,
"step": 234,
- "step_time": 3.8798253620007017
+ "step_time": 10.278297286999987
},
{
"clip_ratio/high_max": 0.0,
@@ -6568,26 +6568,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 60.75,
- "completions/mean_terminated_length": 60.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.425286054611206,
- "epoch": 1.910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2035023272037506,
- "kl": 5.512987627298571e-05,
- "learning_rate": 2.843389687625986e-07,
- "loss": -0.010439477860927582,
- "num_tokens": 962638.0,
- "reward": 0.3157375156879425,
- "reward_std": 0.46047845482826233,
- "rewards/true_env_reward_fn/mean": 0.3157375156879425,
- "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 64.47917175292969,
+ "completions/mean_terminated_length": 64.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3364675641059875,
+ "epoch": 5.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09685558825731277,
+ "kl": 0.00013002969353692606,
+ "learning_rate": 4.638479189738224e-07,
+ "loss": 0.05070740357041359,
+ "num_tokens": 5804436.0,
+ "reward": 0.44921523332595825,
+ "reward_std": 0.22693434357643127,
+ "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+ "rewards/true_env_reward_fn/std": 0.22693434357643127,
"step": 235,
- "step_time": 4.114513064998391
+ "step_time": 10.754199091999908
},
{
"clip_ratio/high_max": 0.0,
@@ -6596,26 +6596,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 120.0,
- "completions/max_terminated_length": 120.0,
- "completions/mean_length": 77.625,
- "completions/mean_terminated_length": 77.625,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4385854601860046,
- "epoch": 1.91869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12982772290706635,
- "kl": 1.2591926861205138e-05,
- "learning_rate": 2.807208355813339e-07,
- "loss": 0.09093751758337021,
- "num_tokens": 965755.0,
- "reward": 0.6140732765197754,
- "reward_std": 0.27462607622146606,
- "rewards/true_env_reward_fn/mean": 0.6140732765197754,
- "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2267529368400574,
+ "epoch": 5.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10451620817184448,
+ "kl": 6.754635069228243e-05,
+ "learning_rate": 4.596035217709039e-07,
+ "loss": -0.02925686165690422,
+ "num_tokens": 5826895.0,
+ "reward": 0.49523353576660156,
+ "reward_std": 0.15093794465065002,
+ "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+ "rewards/true_env_reward_fn/std": 0.15093792974948883,
"step": 236,
- "step_time": 5.147667763001664
+ "step_time": 9.666070583999954
},
{
"clip_ratio/high_max": 0.0,
@@ -6624,26 +6624,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 71.375,
- "completions/mean_terminated_length": 71.375,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.3476852178573608,
- "epoch": 1.9268292682926829,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14480431377887726,
- "kl": 3.8014684832887724e-05,
- "learning_rate": 2.771133827363101e-07,
- "loss": -0.07322391867637634,
- "num_tokens": 970918.0,
- "reward": 0.3427826166152954,
- "reward_std": 0.42430612444877625,
- "rewards/true_env_reward_fn/mean": 0.3427826166152954,
- "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 67.79167175292969,
+ "completions/mean_terminated_length": 67.79167175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2110519707202911,
+ "epoch": 5.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07890205830335617,
+ "kl": 0.0001245876064785989,
+ "learning_rate": 4.5536205268140286e-07,
+ "loss": -0.03693925589323044,
+ "num_tokens": 5859973.0,
+ "reward": 0.2979053258895874,
+ "reward_std": 0.23015481233596802,
+ "rewards/true_env_reward_fn/mean": 0.297905296087265,
+ "rewards/true_env_reward_fn/std": 0.2301548272371292,
"step": 237,
- "step_time": 4.67846887900123
+ "step_time": 13.971699990999241
},
{
"clip_ratio/high_max": 0.0,
@@ -6652,26 +6652,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 221.0,
- "completions/max_terminated_length": 221.0,
- "completions/mean_length": 94.25,
- "completions/mean_terminated_length": 94.25,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.3279914855957031,
- "epoch": 1.934959349593496,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10826019197702408,
- "kl": 1.3074863090878353e-05,
- "learning_rate": 2.7351693324127037e-07,
- "loss": -0.05556309223175049,
- "num_tokens": 975120.0,
- "reward": 0.64573073387146,
- "reward_std": 0.22739914059638977,
- "rewards/true_env_reward_fn/mean": 0.64573073387146,
- "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 65.52083587646484,
+ "completions/mean_terminated_length": 65.52083587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2291057407855988,
+ "epoch": 5.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09152546525001526,
+ "kl": 0.00013101351214572787,
+ "learning_rate": 4.511238191455491e-07,
+ "loss": 0.043641336262226105,
+ "num_tokens": 5886118.0,
+ "reward": 0.41334670782089233,
+ "reward_std": 0.19480590522289276,
+ "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+ "rewards/true_env_reward_fn/std": 0.19480590522289276,
"step": 238,
- "step_time": 9.220254810001279
+ "step_time": 10.805698846999803
},
{
"clip_ratio/high_max": 0.0,
@@ -6680,26 +6680,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3695034384727478,
- "epoch": 1.943089430894309,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12166028469800949,
- "kl": 2.6563114261080045e-05,
- "learning_rate": 2.6993180912471055e-07,
- "loss": -0.05334407091140747,
- "num_tokens": 980254.0,
- "reward": 0.37203267216682434,
- "reward_std": 0.20089927315711975,
- "rewards/true_env_reward_fn/mean": 0.37203267216682434,
- "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 58.833335876464844,
+ "completions/mean_terminated_length": 58.833335876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3134913444519043,
+ "epoch": 5.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07343924045562744,
+ "kl": 0.00019655993492051493,
+ "learning_rate": 4.4688912836904533e-07,
+ "loss": 0.015309082344174385,
+ "num_tokens": 5908334.0,
+ "reward": 0.4438478946685791,
+ "reward_std": 0.27188754081726074,
+ "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+ "rewards/true_env_reward_fn/std": 0.27188754081726074,
"step": 239,
- "step_time": 4.224964968001586
+ "step_time": 9.688736522
},
{
"clip_ratio/high_max": 0.0,
@@ -6708,26 +6708,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.75,
- "completions/mean_terminated_length": 53.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.247464120388031,
- "epoch": 1.951219512195122,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2346416860818863,
- "kl": 5.1520751185307745e-05,
- "learning_rate": 2.6635833140104405e-07,
- "loss": -0.0905834436416626,
- "num_tokens": 983244.0,
- "reward": 0.575507640838623,
- "reward_std": 0.17391785979270935,
- "rewards/true_env_reward_fn/mean": 0.575507640838623,
- "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "completions/max_length": 464.0,
+ "completions/max_terminated_length": 464.0,
+ "completions/mean_length": 79.14583587646484,
+ "completions/mean_terminated_length": 79.14583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3148745000362396,
+ "epoch": 5.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06567981839179993,
+ "kl": 4.445325248525478e-05,
+ "learning_rate": 4.4265828730079977e-07,
+ "loss": 0.021218176931142807,
+ "num_tokens": 5929645.0,
+ "reward": 0.5866342186927795,
+ "reward_std": 0.13780196011066437,
+ "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+ "rewards/true_env_reward_fn/std": 0.13780196011066437,
"step": 240,
- "step_time": 3.357481237999309
+ "step_time": 27.046819901000163
},
{
"clip_ratio/high_max": 0.0,
@@ -6736,26 +6736,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2175387144088745,
- "epoch": 1.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1870066374540329,
- "kl": 2.2185965462995227e-05,
- "learning_rate": 2.6279682004185894e-07,
- "loss": -0.07365687191486359,
- "num_tokens": 985574.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 364.0,
+ "completions/max_terminated_length": 364.0,
+ "completions/mean_length": 70.33333587646484,
+ "completions/mean_terminated_length": 70.33333587646484,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2104995846748352,
+ "epoch": 5.878048780487805,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.042626503854990005,
+ "kl": 6.413207393052289e-05,
+ "learning_rate": 4.3843160261067653e-07,
+ "loss": -0.0065308245830237865,
+ "num_tokens": 5950005.0,
+ "reward": 0.5593677759170532,
+ "reward_std": 0.23993276059627533,
+ "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+ "rewards/true_env_reward_fn/std": 0.23993274569511414,
"step": 241,
- "step_time": 4.527591582998866
+ "step_time": 22.238758486000734
},
{
"clip_ratio/high_max": 0.0,
@@ -6764,26 +6764,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.25,
- "completions/mean_terminated_length": 76.25,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2727615237236023,
- "epoch": 1.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.226071804529056e-05,
- "kl": 1.4014385214977665e-05,
- "learning_rate": 2.592475939472668e-07,
- "loss": 7.015369192231447e-07,
- "num_tokens": 988868.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2132116258144379,
+ "epoch": 5.902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05718924477696419,
+ "kl": 6.140609366411809e-05,
+ "learning_rate": 4.342093806672678e-07,
+ "loss": 0.012734346091747284,
+ "num_tokens": 5974626.0,
+ "reward": 0.4349059462547302,
+ "reward_std": 0.24915602803230286,
+ "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+ "rewards/true_env_reward_fn/std": 0.24915601313114166,
"step": 242,
- "step_time": 4.238274277000528
+ "step_time": 9.239750460000323
},
{
"clip_ratio/high_max": 0.0,
@@ -6791,27 +6791,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1519948840141296,
- "epoch": 1.975609756097561,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12447996437549591,
- "kl": 4.700180943473242e-05,
- "learning_rate": 2.557109709173482e-07,
- "loss": 0.08819369971752167,
- "num_tokens": 994068.0,
- "reward": 0.3696666657924652,
- "reward_std": 0.18488828837871552,
- "rewards/true_env_reward_fn/mean": 0.3696666657924652,
- "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 67.04167175292969,
+ "completions/mean_terminated_length": 57.574466705322266,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.7212344706058502,
+ "epoch": 5.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08872146159410477,
+ "kl": 0.00011613740025495645,
+ "learning_rate": 4.2999192751568557e-07,
+ "loss": -0.021363887935876846,
+ "num_tokens": 5997300.0,
+ "reward": 0.4436037540435791,
+ "reward_std": 0.28323379158973694,
+ "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+ "rewards/true_env_reward_fn/std": 0.28323376178741455,
"step": 243,
- "step_time": 3.3367313500002638
+ "step_time": 29.767976787999487
},
{
"clip_ratio/high_max": 0.0,
@@ -6820,26 +6820,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 115.0,
- "completions/max_terminated_length": 115.0,
- "completions/mean_length": 68.25,
- "completions/mean_terminated_length": 68.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0537148416042328,
- "epoch": 1.9837398373983741,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17681685090065002,
- "kl": 3.6240851841284893e-05,
- "learning_rate": 2.521872676236972e-07,
- "loss": 0.05281040072441101,
- "num_tokens": 1000650.0,
- "reward": 0.12209999561309814,
- "reward_std": 0.2502918243408203,
- "rewards/true_env_reward_fn/mean": 0.12209999561309814,
- "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.083335876464844,
+ "completions/mean_terminated_length": 61.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2353634238243103,
+ "epoch": 5.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07658001780509949,
+ "kl": 9.086773115996039e-05,
+ "learning_rate": 4.257795488553798e-07,
+ "loss": -0.005057391710579395,
+ "num_tokens": 6021752.0,
+ "reward": 0.49971556663513184,
+ "reward_std": 0.2643933594226837,
+ "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+ "rewards/true_env_reward_fn/std": 0.2643933594226837,
"step": 244,
- "step_time": 5.2957401019993995
+ "step_time": 9.764708648999658
},
{
"clip_ratio/high_max": 0.0,
@@ -6847,27 +6847,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 140.0,
- "completions/max_terminated_length": 140.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 61.0,
- "completions/min_terminated_length": 61.0,
- "entropy": 1.3126497864723206,
- "epoch": 1.9918699186991868,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16036106646060944,
- "kl": 6.061139720259234e-05,
- "learning_rate": 2.48676799581066e-07,
- "loss": 0.09418506920337677,
- "num_tokens": 1005737.0,
- "reward": 0.31175702810287476,
- "reward_std": 0.38867074251174927,
- "rewards/true_env_reward_fn/mean": 0.31175702810287476,
- "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 86.72917175292969,
+ "completions/mean_terminated_length": 77.68084716796875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2082330882549286,
+ "epoch": 5.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.059102799743413925,
+ "kl": 5.0710960749711376e-05,
+ "learning_rate": 4.215725500179787e-07,
+ "loss": 0.17436102032661438,
+ "num_tokens": 6049747.0,
+ "reward": 0.46306928992271423,
+ "reward_std": 0.26072925329208374,
+ "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+ "rewards/true_env_reward_fn/std": 0.26072925329208374,
"step": 245,
- "step_time": 6.259088058999623
+ "step_time": 31.997988874000384
},
{
"clip_ratio/high_max": 0.0,
@@ -6876,26 +6876,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 48.5,
- "completions/mean_terminated_length": 48.5,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.21333646774292,
- "epoch": 2.0,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23997871577739716,
- "kl": 3.1378609492094256e-05,
- "learning_rate": 2.4517988111911313e-07,
- "loss": 0.010592922568321228,
- "num_tokens": 1010869.0,
- "reward": 0.33381664752960205,
- "reward_std": 0.18213039636611938,
- "rewards/true_env_reward_fn/mean": 0.33381664752960205,
- "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 61.35416793823242,
+ "completions/mean_terminated_length": 61.35416793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2454268038272858,
+ "epoch": 6.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07332625985145569,
+ "kl": 0.0001107546740968246,
+ "learning_rate": 4.1737123594515755e-07,
+ "loss": -0.013426866382360458,
+ "num_tokens": 6072668.0,
+ "reward": 0.45415109395980835,
+ "reward_std": 0.2937946617603302,
+ "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+ "rewards/true_env_reward_fn/std": 0.2937946617603302,
"step": 246,
- "step_time": 3.111915630997828
+ "step_time": 9.587768273000165
},
{
"clip_ratio/high_max": 0.0,
@@ -6904,26 +6904,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.3962982892990112,
- "epoch": 2.008130081300813,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21355785429477692,
- "kl": 3.789625407080166e-05,
- "learning_rate": 2.4169682535425927e-07,
- "loss": 0.025682777166366577,
- "num_tokens": 1014876.0,
- "reward": 0.35749268531799316,
- "reward_std": 0.29738906025886536,
- "rewards/true_env_reward_fn/mean": 0.35749268531799316,
- "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 71.5625,
+ "completions/mean_terminated_length": 71.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1738699078559875,
+ "epoch": 6.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06093747541308403,
+ "kl": 7.893411338955048e-05,
+ "learning_rate": 4.131759111665348e-07,
+ "loss": 0.022837676107883453,
+ "num_tokens": 6095263.0,
+ "reward": 0.5188159346580505,
+ "reward_std": 0.3265886902809143,
+ "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+ "rewards/true_env_reward_fn/std": 0.3265886902809143,
"step": 247,
- "step_time": 3.35338095500083
+ "step_time": 14.232978527999876
},
{
"clip_ratio/high_max": 0.0,
@@ -6932,26 +6932,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 73.875,
- "completions/mean_terminated_length": 73.875,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.463137686252594,
- "epoch": 2.016260162601626,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18188992142677307,
- "kl": 1.746804719005013e-05,
- "learning_rate": 2.382279441616492e-07,
- "loss": -0.17857304215431213,
- "num_tokens": 1018383.0,
- "reward": 0.5329012274742126,
- "reward_std": 0.055823445320129395,
- "rewards/true_env_reward_fn/mean": 0.5329012274742126,
- "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3300949931144714,
+ "epoch": 6.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08098509907722473,
+ "kl": 0.00010828049380506855,
+ "learning_rate": 4.0898687977759887e-07,
+ "loss": 0.01631344109773636,
+ "num_tokens": 6125380.0,
+ "reward": 0.3338983952999115,
+ "reward_std": 0.19050820171833038,
+ "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+ "rewards/true_env_reward_fn/std": 0.19050820171833038,
"step": 248,
- "step_time": 5.210386754000865
+ "step_time": 13.248441182000079
},
{
"clip_ratio/high_max": 0.0,
@@ -6960,26 +6960,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 79.25,
- "completions/mean_terminated_length": 79.25,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.4478936195373535,
- "epoch": 2.024390243902439,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.744662434561178e-05,
- "kl": 1.3336490155779757e-05,
- "learning_rate": 2.3477354814722762e-07,
- "loss": 6.725406365148956e-07,
- "num_tokens": 1022753.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 396.0,
+ "completions/max_terminated_length": 396.0,
+ "completions/mean_length": 88.14583587646484,
+ "completions/mean_terminated_length": 88.14583587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3161276876926422,
+ "epoch": 6.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07784765958786011,
+ "kl": 4.819030800717883e-05,
+ "learning_rate": 4.0480444541766575e-07,
+ "loss": 0.12299837917089462,
+ "num_tokens": 6151955.0,
+ "reward": 0.5446512699127197,
+ "reward_std": 0.2611033618450165,
+ "rewards/true_env_reward_fn/mean": 0.544651210308075,
+ "rewards/true_env_reward_fn/std": 0.2611033618450165,
"step": 249,
- "step_time": 8.798317029002646
+ "step_time": 25.232192139999825
},
{
"clip_ratio/high_max": 0.0,
@@ -6988,26 +6988,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 64.75,
- "completions/mean_terminated_length": 64.75,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.37166029214859,
- "epoch": 2.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1542455554008484,
- "kl": 2.0379054603836266e-05,
- "learning_rate": 2.313339466199264e-07,
- "loss": -0.037539318203926086,
- "num_tokens": 1025971.0,
- "reward": 0.6065863966941833,
- "reward_std": 0.032470256090164185,
- "rewards/true_env_reward_fn/mean": 0.6065863966941833,
- "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 69.97917175292969,
+ "completions/mean_terminated_length": 69.97917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2318329215049744,
+ "epoch": 6.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06498592346906662,
+ "kl": 3.854301621686318e-05,
+ "learning_rate": 4.0062891124787e-07,
+ "loss": -0.04302535578608513,
+ "num_tokens": 6174898.0,
+ "reward": 0.6081289052963257,
+ "reward_std": 0.24437586963176727,
+ "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+ "rewards/true_env_reward_fn/std": 0.24437588453292847,
"step": 250,
- "step_time": 4.096263454999644
+ "step_time": 10.385816780999448
},
{
"clip_ratio/high_max": 0.0,
@@ -7016,26 +7016,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 60.5,
- "completions/mean_terminated_length": 60.5,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4042693972587585,
- "epoch": 2.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.163822203874588,
- "kl": 3.525477495713858e-05,
- "learning_rate": 2.2790944756396916e-07,
- "loss": 0.03408379852771759,
- "num_tokens": 1029415.0,
- "reward": 0.37829869985580444,
- "reward_std": 0.2773255407810211,
- "rewards/true_env_reward_fn/mean": 0.37829869985580444,
- "rewards/true_env_reward_fn/std": 0.2773255407810211,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.91666793823242,
+ "completions/mean_terminated_length": 60.91666793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2425517737865448,
+ "epoch": 6.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08402577042579651,
+ "kl": 0.00010545800250838511,
+ "learning_rate": 3.9646057992918966e-07,
+ "loss": 0.006328321993350983,
+ "num_tokens": 6203582.0,
+ "reward": 0.30250340700149536,
+ "reward_std": 0.3084425926208496,
+ "rewards/true_env_reward_fn/mean": 0.30250340700149536,
+ "rewards/true_env_reward_fn/std": 0.3084425628185272,
"step": 251,
- "step_time": 3.9464334140011488
+ "step_time": 9.779451584999606
},
{
"clip_ratio/high_max": 0.0,
@@ -7044,26 +7044,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 181.0,
- "completions/max_terminated_length": 181.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2268111109733582,
- "epoch": 2.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1961166262626648,
- "kl": 3.528672823449597e-05,
- "learning_rate": 2.2450035761129391e-07,
- "loss": 0.2999379336833954,
- "num_tokens": 1032213.0,
- "reward": 0.6772161722183228,
- "reward_std": 0.31218820810317993,
- "rewards/true_env_reward_fn/mean": 0.6772161722183228,
- "rewards/true_env_reward_fn/std": 0.31218820810317993,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.5,
+ "completions/mean_terminated_length": 65.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2977190017700195,
+ "epoch": 6.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372989296913147,
+ "kl": 0.00017708011364447884,
+ "learning_rate": 3.9229975360050934e-07,
+ "loss": 0.04882372170686722,
+ "num_tokens": 6227486.0,
+ "reward": 0.5265982151031494,
+ "reward_std": 0.31736499071121216,
+ "rewards/true_env_reward_fn/mean": 0.5265981554985046,
+ "rewards/true_env_reward_fn/std": 0.31736496090888977,
"step": 252,
- "step_time": 7.437029113001699
+ "step_time": 10.318136508000407
},
{
"clip_ratio/high_max": 0.0,
@@ -7072,26 +7072,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.360656499862671,
- "epoch": 2.0569105691056913,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24334783852100372,
- "kl": 5.1042834456893615e-05,
- "learning_rate": 2.2110698201409787e-07,
- "loss": 0.05509951710700989,
- "num_tokens": 1038598.0,
- "reward": 0.2947666645050049,
- "reward_std": 0.19891902804374695,
- "rewards/true_env_reward_fn/mean": 0.2947666645050049,
- "rewards/true_env_reward_fn/std": 0.19891902804374695,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 71.125,
+ "completions/mean_terminated_length": 71.125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3135383129119873,
+ "epoch": 6.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05451377481222153,
+ "kl": 6.562464477610774e-05,
+ "learning_rate": 3.8814673385671893e-07,
+ "loss": 0.01392640545964241,
+ "num_tokens": 6255436.0,
+ "reward": 0.4130678176879883,
+ "reward_std": 0.1573377549648285,
+ "rewards/true_env_reward_fn/mean": 0.4130678176879883,
+ "rewards/true_env_reward_fn/std": 0.15733776986598969,
"step": 253,
- "step_time": 3.7359043900014512
+ "step_time": 9.396596211999622
},
{
"clip_ratio/high_max": 0.0,
@@ -7100,26 +7100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 74.75,
- "completions/mean_terminated_length": 74.75,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4314632415771484,
- "epoch": 2.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1960419863462448,
- "kl": 6.17889963905327e-05,
- "learning_rate": 2.1772962461750342e-07,
- "loss": 0.0954262986779213,
- "num_tokens": 1043268.0,
- "reward": 0.4036714732646942,
- "reward_std": 0.42137831449508667,
- "rewards/true_env_reward_fn/mean": 0.4036714732646942,
- "rewards/true_env_reward_fn/std": 0.42137834429740906,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 73.85417175292969,
+ "completions/mean_terminated_length": 73.85417175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3749704957008362,
+ "epoch": 6.195121951219512,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05664386227726936,
+ "kl": 6.744195252395002e-05,
+ "learning_rate": 3.840018217268526e-07,
+ "loss": -0.04205852374434471,
+ "num_tokens": 6281557.0,
+ "reward": 0.5188004970550537,
+ "reward_std": 0.2932124733924866,
+ "rewards/true_env_reward_fn/mean": 0.5188004970550537,
+ "rewards/true_env_reward_fn/std": 0.2932124435901642,
"step": 254,
- "step_time": 5.565175547999388
+ "step_time": 19.77463799499992
},
{
"clip_ratio/high_max": 0.0,
@@ -7128,26 +7128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0521443486213684,
- "epoch": 2.073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.464648271910846e-05,
- "kl": 1.2018902907584561e-05,
- "learning_rate": 2.1436858783235338e-07,
- "loss": 6.008343689245521e-07,
- "num_tokens": 1046517.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2866049408912659,
+ "epoch": 6.219512195121951,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06639927625656128,
+ "kl": 5.235667686065426e-05,
+ "learning_rate": 3.798653176522696e-07,
+ "loss": -0.019558893516659737,
+ "num_tokens": 6302850.0,
+ "reward": 0.5025100111961365,
+ "reward_std": 0.1717289537191391,
+ "rewards/true_env_reward_fn/mean": 0.5025100111961365,
+ "rewards/true_env_reward_fn/std": 0.1717289537191391,
"step": 255,
- "step_time": 4.379171047001364
+ "step_time": 9.634558264001043
},
{
"clip_ratio/high_max": 0.0,
@@ -7156,26 +7156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.091518223285675,
- "epoch": 2.08130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.263061656383798e-05,
- "kl": 1.4573892713087844e-05,
- "learning_rate": 2.110241726081317e-07,
- "loss": 7.304333848878741e-07,
- "num_tokens": 1050439.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.3105897009372711,
+ "epoch": 6.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07659970223903656,
+ "kl": 8.12946273072157e-05,
+ "learning_rate": 3.7573752146487636e-07,
+ "loss": 0.05201243981719017,
+ "num_tokens": 6329269.0,
+ "reward": 0.5027985572814941,
+ "reward_std": 0.2521378695964813,
+ "rewards/true_env_reward_fn/mean": 0.5027985572814941,
+ "rewards/true_env_reward_fn/std": 0.2521378993988037,
"step": 256,
- "step_time": 3.8379976090000127
+ "step_time": 11.07390475200009
},
{
"clip_ratio/high_max": 0.0,
@@ -7184,26 +7184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2301559448242188,
- "epoch": 2.089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17180238664150238,
- "kl": 2.236898035334889e-05,
- "learning_rate": 2.076966784060165e-07,
- "loss": 0.023751959204673767,
- "num_tokens": 1054578.0,
- "reward": 0.4112047851085663,
- "reward_std": 0.05330020561814308,
- "rewards/true_env_reward_fn/mean": 0.4112047851085663,
- "rewards/true_env_reward_fn/std": 0.05330020561814308,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.3154918253421783,
+ "epoch": 6.2682926829268295,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10612925887107849,
+ "kl": 0.00010100230429088697,
+ "learning_rate": 3.7161873236539386e-07,
+ "loss": 0.0214182510972023,
+ "num_tokens": 6356233.0,
+ "reward": 0.2817384600639343,
+ "reward_std": 0.3363598883152008,
+ "rewards/true_env_reward_fn/mean": 0.2817384600639343,
+ "rewards/true_env_reward_fn/std": 0.3363598883152008,
"step": 257,
- "step_time": 4.424114469000415
+ "step_time": 10.784447634000117
},
{
"clip_ratio/high_max": 0.0,
@@ -7212,26 +7212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 52.125,
- "completions/mean_terminated_length": 52.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2734522223472595,
- "epoch": 2.097560975609756,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2274128645658493,
- "kl": 5.3426387239596806e-05,
- "learning_rate": 2.043864031720667e-07,
- "loss": 0.05165906995534897,
- "num_tokens": 1060579.0,
- "reward": 0.2129499912261963,
- "reward_std": 0.2476053088903427,
- "rewards/true_env_reward_fn/mean": 0.2129499912261963,
- "rewards/true_env_reward_fn/std": 0.2476053088903427,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 62.770835876464844,
+ "completions/mean_terminated_length": 62.770835876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1986172497272491,
+ "epoch": 6.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08129199594259262,
+ "kl": 0.00011449725025158841,
+ "learning_rate": 3.6750924890166923e-07,
+ "loss": -0.05941678211092949,
+ "num_tokens": 6375046.0,
+ "reward": 0.5760313272476196,
+ "reward_std": 0.24430812895298004,
+ "rewards/true_env_reward_fn/mean": 0.5760312676429749,
+ "rewards/true_env_reward_fn/std": 0.24430814385414124,
"step": 258,
- "step_time": 3.4830677139998443
+ "step_time": 9.467202022999572
},
{
"clip_ratio/high_max": 0.0,
@@ -7240,26 +7240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 54.375,
- "completions/mean_terminated_length": 54.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2013150453567505,
- "epoch": 2.105691056910569,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20796725153923035,
- "kl": 8.34841157484334e-05,
- "learning_rate": 2.0109364331054297e-07,
- "loss": 0.04251064360141754,
- "num_tokens": 1065318.0,
- "reward": 0.2038009911775589,
- "reward_std": 0.3393669128417969,
- "rewards/true_env_reward_fn/mean": 0.2038009911775589,
- "rewards/true_env_reward_fn/std": 0.3393669128417969,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2520000636577606,
+ "epoch": 6.317073170731708,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.06876237690448761,
+ "kl": 0.0001509120993432589,
+ "learning_rate": 3.6340936894703713e-07,
+ "loss": 0.015932239592075348,
+ "num_tokens": 6397298.0,
+ "reward": 0.4944794774055481,
+ "reward_std": 0.24476772546768188,
+ "rewards/true_env_reward_fn/mean": 0.4944794476032257,
+ "rewards/true_env_reward_fn/std": 0.24476774036884308,
"step": 259,
- "step_time": 3.34712773299907
+ "step_time": 10.47640546699995
},
{
"clip_ratio/high_max": 0.0,
@@ -7268,26 +7268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 52.25,
- "completions/mean_terminated_length": 52.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3005307912826538,
- "epoch": 2.113821138211382,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2342396229505539,
- "kl": 2.9608699151140172e-05,
- "learning_rate": 1.9781869365736777e-07,
- "loss": -0.11542908847332001,
- "num_tokens": 1068352.0,
- "reward": 0.5755212306976318,
- "reward_std": 0.1629202961921692,
- "rewards/true_env_reward_fn/mean": 0.5755212306976318,
- "rewards/true_env_reward_fn/std": 0.1629202961921692,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3675504922866821,
+ "epoch": 6.341463414634147,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06423573940992355,
+ "kl": 0.00010545238592385431,
+ "learning_rate": 3.593193896787277e-07,
+ "loss": 0.006066613830626011,
+ "num_tokens": 6423927.0,
+ "reward": 0.4633293151855469,
+ "reward_std": 0.2953638732433319,
+ "rewards/true_env_reward_fn/mean": 0.4633293151855469,
+ "rewards/true_env_reward_fn/std": 0.2953638732433319,
"step": 260,
- "step_time": 3.0931306170004973
+ "step_time": 11.672983966999709
},
{
"clip_ratio/high_max": 0.0,
@@ -7296,26 +7296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 70.875,
- "completions/mean_terminated_length": 70.875,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1381222009658813,
- "epoch": 2.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13169968128204346,
- "kl": 1.4705466128361877e-05,
- "learning_rate": 1.9456184745372558e-07,
- "loss": -0.026440951973199844,
- "num_tokens": 1070847.0,
- "reward": 0.5704532265663147,
- "reward_std": 0.13928835093975067,
- "rewards/true_env_reward_fn/mean": 0.5704532265663147,
- "rewards/true_env_reward_fn/std": 0.13928835093975067,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 74.5,
+ "completions/mean_terminated_length": 74.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3533118963241577,
+ "epoch": 6.365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07830996066331863,
+ "kl": 8.717951732251095e-05,
+ "learning_rate": 3.552396075563257e-07,
+ "loss": -0.021772567182779312,
+ "num_tokens": 6451487.0,
+ "reward": 0.3543766736984253,
+ "reward_std": 0.33776554465293884,
+ "rewards/true_env_reward_fn/mean": 0.3543766736984253,
+ "rewards/true_env_reward_fn/std": 0.33776557445526123,
"step": 261,
- "step_time": 3.8980969309996
+ "step_time": 13.734938852000141
},
{
"clip_ratio/high_max": 0.0,
@@ -7324,26 +7324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.241390347480774,
- "epoch": 2.130081300813008,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010450400441186503,
- "kl": 1.6406540453317575e-05,
- "learning_rate": 1.9132339631980622e-07,
- "loss": 8.202600838558283e-07,
- "num_tokens": 1076153.0,
- "reward": 0.41493332386016846,
- "reward_std": 0.06228968873620033,
- "rewards/true_env_reward_fn/mean": 0.41493332386016846,
- "rewards/true_env_reward_fn/std": 0.06228969246149063,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 71.45833587646484,
+ "completions/mean_terminated_length": 71.45833587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1788119971752167,
+ "epoch": 6.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0994381383061409,
+ "kl": 0.00011037426884286106,
+ "learning_rate": 3.511703183002827e-07,
+ "loss": 0.008012240752577782,
+ "num_tokens": 6476141.0,
+ "reward": 0.45457857847213745,
+ "reward_std": 0.24262367188930511,
+ "rewards/true_env_reward_fn/mean": 0.45457854866981506,
+ "rewards/true_env_reward_fn/std": 0.24262367188930511,
"step": 262,
- "step_time": 3.6601423579995753
+ "step_time": 10.59573544200066
},
{
"clip_ratio/high_max": 0.0,
@@ -7352,26 +7352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 81.0,
- "completions/mean_terminated_length": 81.0,
- "completions/min_length": 67.0,
- "completions/min_terminated_length": 67.0,
- "entropy": 1.1326860189437866,
- "epoch": 2.138211382113821,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001754833065206185,
- "kl": 1.4390577689482598e-05,
- "learning_rate": 1.881036302286923e-07,
- "loss": 7.2446778176527e-07,
- "num_tokens": 1080537.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.293170690536499,
+ "epoch": 6.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10550480335950851,
+ "kl": 0.00020241059792169835,
+ "learning_rate": 3.4711181687048106e-07,
+ "loss": 0.02680305391550064,
+ "num_tokens": 6499670.0,
+ "reward": 0.33261698484420776,
+ "reward_std": 0.31543657183647156,
+ "rewards/true_env_reward_fn/mean": 0.3326169550418854,
+ "rewards/true_env_reward_fn/std": 0.31543657183647156,
"step": 263,
- "step_time": 4.190891189999093
+ "step_time": 10.02452396199942
},
{
"clip_ratio/high_max": 0.0,
@@ -7380,26 +7380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 103.0,
- "completions/max_terminated_length": 103.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.353486955165863,
- "epoch": 2.1463414634146343,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.215454563498497,
- "kl": 6.859865607111715e-05,
- "learning_rate": 1.8490283748039515e-07,
- "loss": 0.0181141197681427,
- "num_tokens": 1084056.0,
- "reward": 0.3049938380718231,
- "reward_std": 0.4605039656162262,
- "rewards/true_env_reward_fn/mean": 0.3049938380718231,
- "rewards/true_env_reward_fn/std": 0.4605039954185486,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 89.83333587646484,
+ "completions/mean_terminated_length": 89.83333587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2982321977615356,
+ "epoch": 6.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06412192434072495,
+ "kl": 6.536830551340245e-05,
+ "learning_rate": 3.4306439744485447e-07,
+ "loss": -0.08396965265274048,
+ "num_tokens": 6522630.0,
+ "reward": 0.5518762469291687,
+ "reward_std": 0.21314994990825653,
+ "rewards/true_env_reward_fn/mean": 0.5518762469291687,
+ "rewards/true_env_reward_fn/std": 0.21314994990825653,
"step": 264,
- "step_time": 4.504906432999633
+ "step_time": 20.82168071200067
},
{
"clip_ratio/high_max": 0.0,
@@ -7408,26 +7408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 127.0,
- "completions/max_terminated_length": 127.0,
- "completions/mean_length": 70.375,
- "completions/mean_terminated_length": 70.375,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 63.85416793823242,
+ "completions/mean_terminated_length": 63.85416793823242,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.0113105773925781,
- "epoch": 2.154471544715447,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10072485357522964,
- "kl": 1.4604076568502933e-05,
- "learning_rate": 1.8172130467604085e-07,
- "loss": -0.041721273213624954,
- "num_tokens": 1090171.0,
- "reward": 0.47745320200920105,
- "reward_std": 0.2929421067237854,
- "rewards/true_env_reward_fn/mean": 0.47745320200920105,
- "rewards/true_env_reward_fn/std": 0.2929421067237854,
+ "entropy": 1.2279660403728485,
+ "epoch": 6.463414634146342,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10119301080703735,
+ "kl": 0.00010847221983567579,
+ "learning_rate": 3.3902835339806456e-07,
+ "loss": 0.01589711755514145,
+ "num_tokens": 6548183.0,
+ "reward": 0.4527897238731384,
+ "reward_std": 0.2534019947052002,
+ "rewards/true_env_reward_fn/mean": 0.45278969407081604,
+ "rewards/true_env_reward_fn/std": 0.2534019649028778,
"step": 265,
- "step_time": 6.299696521999067
+ "step_time": 8.63894235699945
},
{
"clip_ratio/high_max": 0.0,
@@ -7436,26 +7436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.375,
- "completions/mean_terminated_length": 57.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3366597294807434,
- "epoch": 2.16260162601626,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2297964245080948,
- "kl": 5.677436638507061e-05,
- "learning_rate": 1.785593166922062e-07,
- "loss": 0.20361776649951935,
- "num_tokens": 1094358.0,
- "reward": 0.3835672438144684,
- "reward_std": 0.4339357614517212,
- "rewards/true_env_reward_fn/mean": 0.3835672438144684,
- "rewards/true_env_reward_fn/std": 0.4339357912540436,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 60.54166793823242,
+ "completions/mean_terminated_length": 60.54166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1612891554832458,
+ "epoch": 6.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0825633853673935,
+ "kl": 7.345602170971688e-05,
+ "learning_rate": 3.3500397728023534e-07,
+ "loss": 0.057398565113544464,
+ "num_tokens": 6571537.0,
+ "reward": 0.5196421146392822,
+ "reward_std": 0.18547315895557404,
+ "rewards/true_env_reward_fn/mean": 0.5196421146392822,
+ "rewards/true_env_reward_fn/std": 0.18547315895557404,
"step": 266,
- "step_time": 4.192992550000781
+ "step_time": 10.01155260700034
},
{
"clip_ratio/high_max": 0.0,
@@ -7464,26 +7464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4040917754173279,
- "epoch": 2.1707317073170733,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2408597469329834,
+ "epoch": 6.512195121951219,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1913405954837799,
- "kl": 2.2514723241329193e-05,
- "learning_rate": 1.7541715665541276e-07,
- "loss": -0.12278837710618973,
- "num_tokens": 1099685.0,
- "reward": 0.5316476821899414,
- "reward_std": 0.1867343932390213,
- "rewards/true_env_reward_fn/mean": 0.5316476821899414,
- "rewards/true_env_reward_fn/std": 0.1867344230413437,
+ "grad_norm": 0.09217255562543869,
+ "kl": 9.46905902310391e-05,
+ "learning_rate": 3.3099156079574867e-07,
+ "loss": 0.026750221848487854,
+ "num_tokens": 6596010.0,
+ "reward": 0.4330406188964844,
+ "reward_std": 0.20423907041549683,
+ "rewards/true_env_reward_fn/mean": 0.4330406188964844,
+ "rewards/true_env_reward_fn/std": 0.20423908531665802,
"step": 267,
- "step_time": 4.577502725998784
+ "step_time": 20.002466699999786
},
{
"clip_ratio/high_max": 0.0,
@@ -7492,26 +7492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2707499265670776,
- "epoch": 2.178861788617886,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1252160668373108,
- "kl": 2.1676510186807718e-05,
- "learning_rate": 1.722951059167737e-07,
- "loss": 0.030697450041770935,
- "num_tokens": 1103244.0,
- "reward": 0.47949954867362976,
- "reward_std": 0.10323704034090042,
- "rewards/true_env_reward_fn/mean": 0.47949954867362976,
- "rewards/true_env_reward_fn/std": 0.10323705524206161,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 64.83333587646484,
+ "completions/mean_terminated_length": 64.83333587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2907497882843018,
+ "epoch": 6.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07903403043746948,
+ "kl": 0.00015101409735507332,
+ "learning_rate": 3.269913947820998e-07,
+ "loss": 0.02006390690803528,
+ "num_tokens": 6622106.0,
+ "reward": 0.4544076919555664,
+ "reward_std": 0.26717478036880493,
+ "rewards/true_env_reward_fn/mean": 0.4544076919555664,
+ "rewards/true_env_reward_fn/std": 0.2671748101711273,
"step": 268,
- "step_time": 4.3898782989999745
+ "step_time": 8.818348709999555
},
{
"clip_ratio/high_max": 0.0,
@@ -7520,26 +7520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3778526186943054,
- "epoch": 2.186991869918699,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19924476742744446,
- "kl": 5.189802323002368e-05,
- "learning_rate": 1.6919344402680231e-07,
- "loss": 0.03015017881989479,
- "num_tokens": 1108318.0,
- "reward": 0.12559716403484344,
- "reward_std": 0.22310735285282135,
- "rewards/true_env_reward_fn/mean": 0.12559716403484344,
- "rewards/true_env_reward_fn/std": 0.22310735285282135,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2150432765483856,
+ "epoch": 6.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09289928525686264,
+ "kl": 9.474463149672374e-05,
+ "learning_rate": 3.230037691888162e-07,
+ "loss": 0.08772514015436172,
+ "num_tokens": 6636101.0,
+ "reward": 0.6059033870697021,
+ "reward_std": 0.23812197148799896,
+ "rewards/true_env_reward_fn/mean": 0.6059033274650574,
+ "rewards/true_env_reward_fn/std": 0.23812197148799896,
"step": 269,
- "step_time": 4.379851057999986
+ "step_time": 8.47666211500109
},
{
"clip_ratio/high_max": 0.0,
@@ -7548,26 +7548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2916911840438843,
- "epoch": 2.1951219512195124,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18381677567958832,
- "kl": 2.593698127384414e-05,
- "learning_rate": 1.6611244871038116e-07,
- "loss": 0.025029506534337997,
- "num_tokens": 1116025.0,
- "reward": 0.05200198292732239,
- "reward_std": 0.276480108499527,
- "rewards/true_env_reward_fn/mean": 0.05200198292732239,
- "rewards/true_env_reward_fn/std": 0.276480108499527,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.2745259404182434,
+ "epoch": 6.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09368392825126648,
+ "kl": 0.0001518711451353738,
+ "learning_rate": 3.1902897305644093e-07,
+ "loss": -0.003839995712041855,
+ "num_tokens": 6659877.0,
+ "reward": 0.5029901266098022,
+ "reward_std": 0.3467065095901489,
+ "rewards/true_env_reward_fn/mean": 0.5029900670051575,
+ "rewards/true_env_reward_fn/std": 0.3467065393924713,
"step": 270,
- "step_time": 3.6788300769985653
+ "step_time": 11.298448464000103
},
{
"clip_ratio/high_max": 0.0,
@@ -7576,26 +7576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 215.0,
- "completions/max_terminated_length": 215.0,
- "completions/mean_length": 94.625,
- "completions/mean_terminated_length": 94.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2625537514686584,
- "epoch": 2.203252032520325,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.09628148376941681,
- "kl": 2.636932003952097e-05,
- "learning_rate": 1.6305239584189344e-07,
- "loss": 0.009904414415359497,
- "num_tokens": 1120434.0,
- "reward": 0.5924437046051025,
- "reward_std": 0.39917245507240295,
- "rewards/true_env_reward_fn/mean": 0.5924437046051025,
- "rewards/true_env_reward_fn/std": 0.39917245507240295,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 67.0625,
+ "completions/mean_terminated_length": 67.0625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.238816499710083,
+ "epoch": 6.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05819084122776985,
+ "kl": 6.655609695371822e-05,
+ "learning_rate": 3.150672944955818e-07,
+ "loss": -0.02771003544330597,
+ "num_tokens": 6679776.0,
+ "reward": 0.5772091746330261,
+ "reward_std": 0.17815756797790527,
+ "rewards/true_env_reward_fn/mean": 0.5772091746330261,
+ "rewards/true_env_reward_fn/std": 0.17815756797790527,
"step": 271,
- "step_time": 9.09279120499923
+ "step_time": 7.902968623000561
},
{
"clip_ratio/high_max": 0.0,
@@ -7604,26 +7604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.2491654753684998,
- "epoch": 2.2113821138211383,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015083610196597874,
- "kl": 1.4522283436235739e-05,
- "learning_rate": 1.6001355942052182e-07,
- "loss": 7.247089683914965e-07,
- "num_tokens": 1122349.0,
- "reward": 0.812765896320343,
- "reward_std": 0.04731824994087219,
- "rewards/true_env_reward_fn/mean": 0.812765896320343,
- "rewards/true_env_reward_fn/std": 0.04731824994087219,
+ "completions/max_length": 145.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 75.10417175292969,
+ "completions/mean_terminated_length": 75.10417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3645851016044617,
+ "epoch": 6.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07474905252456665,
+ "kl": 7.497054593841312e-05,
+ "learning_rate": 3.1111902066602724e-07,
+ "loss": 0.07271203398704529,
+ "num_tokens": 6704717.0,
+ "reward": 0.42504438757896423,
+ "reward_std": 0.2500284016132355,
+ "rewards/true_env_reward_fn/mean": 0.42504438757896423,
+ "rewards/true_env_reward_fn/std": 0.2500284016132355,
"step": 272,
- "step_time": 3.7730076539992297
+ "step_time": 11.259095110999624
},
{
"clip_ratio/high_max": 0.0,
@@ -7632,26 +7632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 71.25,
- "completions/mean_terminated_length": 71.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1462301015853882,
- "epoch": 2.2195121951219514,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00012917000276502222,
- "kl": 1.3649782886204775e-05,
- "learning_rate": 1.569962115457138e-07,
- "loss": 6.695274805679219e-07,
- "num_tokens": 1124831.0,
- "reward": 0.7329437732696533,
- "reward_std": 0.22123214602470398,
- "rewards/true_env_reward_fn/mean": 0.7329437732696533,
- "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 67.64583587646484,
+ "completions/mean_terminated_length": 67.64583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2410458326339722,
+ "epoch": 6.658536585365853,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0750170573592186,
+ "kl": 8.587932643422391e-05,
+ "learning_rate": 3.0718443775593225e-07,
+ "loss": -0.019169267266988754,
+ "num_tokens": 6727060.0,
+ "reward": 0.5114523768424988,
+ "reward_std": 0.17165428400039673,
+ "rewards/true_env_reward_fn/mean": 0.5114523768424988,
+ "rewards/true_env_reward_fn/std": 0.17165428400039673,
"step": 273,
- "step_time": 3.907510233000721
+ "step_time": 11.097374408000178
},
{
"clip_ratio/high_max": 0.0,
@@ -7660,26 +7660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 175.0,
- "completions/max_terminated_length": 175.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2922418713569641,
- "epoch": 2.227642276422764,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15563185513019562,
- "kl": 3.604595076467376e-05,
- "learning_rate": 1.5400062239281858e-07,
- "loss": -0.034219659864902496,
- "num_tokens": 1128837.0,
- "reward": 0.46255773305892944,
- "reward_std": 0.24893923103809357,
- "rewards/true_env_reward_fn/mean": 0.46255773305892944,
- "rewards/true_env_reward_fn/std": 0.24893923103809357,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 67.08333587646484,
+ "completions/mean_terminated_length": 67.08333587646484,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2417791783809662,
+ "epoch": 6.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08969064801931381,
+ "kl": 5.61167857995315e-05,
+ "learning_rate": 3.0326383096107423e-07,
+ "loss": 0.11341358721256256,
+ "num_tokens": 6752824.0,
+ "reward": 0.4772833287715912,
+ "reward_std": 0.31618404388427734,
+ "rewards/true_env_reward_fn/mean": 0.4772833287715912,
+ "rewards/true_env_reward_fn/std": 0.31618407368659973,
"step": 274,
- "step_time": 7.502110859999448
+ "step_time": 11.189890726001067
},
{
"clip_ratio/high_max": 0.0,
@@ -7688,26 +7688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2709790468215942,
- "epoch": 2.2357723577235773,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.47917175292969,
+ "completions/mean_terminated_length": 67.47917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.272821843624115,
+ "epoch": 6.7073170731707314,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.24958185851573944,
- "kl": 6.180045966175385e-05,
- "learning_rate": 1.5102706018889428e-07,
- "loss": -0.10148808360099792,
- "num_tokens": 1132891.0,
- "reward": 0.4240284562110901,
- "reward_std": 0.43752968311309814,
- "rewards/true_env_reward_fn/mean": 0.4240284562110901,
- "rewards/true_env_reward_fn/std": 0.43752965331077576,
+ "grad_norm": 0.08874963968992233,
+ "kl": 0.00015055539006425533,
+ "learning_rate": 2.9935748446418065e-07,
+ "loss": 0.02028803899884224,
+ "num_tokens": 6776703.0,
+ "reward": 0.49093031883239746,
+ "reward_std": 0.25163188576698303,
+ "rewards/true_env_reward_fn/mean": 0.49093031883239746,
+ "rewards/true_env_reward_fn/std": 0.25163188576698303,
"step": 275,
- "step_time": 3.251475233999372
+ "step_time": 9.941926390000845
},
{
"clip_ratio/high_max": 0.0,
@@ -7716,26 +7716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 55.5,
- "completions/mean_terminated_length": 55.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2637454867362976,
- "epoch": 2.2439024390243905,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.439944602083415e-05,
- "kl": 1.127877567341784e-05,
- "learning_rate": 1.4807579118869146e-07,
- "loss": 5.64579522688291e-07,
- "num_tokens": 1137611.0,
- "reward": 0.4544333219528198,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4544333219528198,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 365.0,
+ "completions/max_terminated_length": 365.0,
+ "completions/mean_length": 68.04167175292969,
+ "completions/mean_terminated_length": 68.04167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0912510752677917,
+ "epoch": 6.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060312703251838684,
+ "kl": 6.840269179519964e-05,
+ "learning_rate": 2.9546568141433e-07,
+ "loss": -0.036469291895627975,
+ "num_tokens": 6799417.0,
+ "reward": 0.5543485879898071,
+ "reward_std": 0.22147472202777863,
+ "rewards/true_env_reward_fn/mean": 0.5543485283851624,
+ "rewards/true_env_reward_fn/std": 0.22147469222545624,
"step": 276,
- "step_time": 3.311975311999049
+ "step_time": 22.291117544999906
},
{
"clip_ratio/high_max": 0.0,
@@ -7744,26 +7744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1065265536308289,
- "epoch": 2.252032520325203,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13068579137325287,
- "kl": 4.4293181417742744e-05,
- "learning_rate": 1.4514707965081262e-07,
- "loss": 0.04848391190171242,
- "num_tokens": 1142087.0,
- "reward": 0.2852628827095032,
- "reward_std": 0.23009054362773895,
- "rewards/true_env_reward_fn/mean": 0.2852628827095032,
- "rewards/true_env_reward_fn/std": 0.23009057343006134,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.895835876464844,
+ "completions/mean_terminated_length": 59.895835876464844,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.2566059231758118,
+ "epoch": 6.7560975609756095,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09982399642467499,
+ "kl": 0.00020091429905733094,
+ "learning_rate": 2.9158870390642863e-07,
+ "loss": -0.005738064646720886,
+ "num_tokens": 6829804.0,
+ "reward": 0.24754562973976135,
+ "reward_std": 0.23038579523563385,
+ "rewards/true_env_reward_fn/mean": 0.24754561483860016,
+ "rewards/true_env_reward_fn/std": 0.23038578033447266,
"step": 277,
- "step_time": 3.9521008399988204
+ "step_time": 9.732460060999983
},
{
"clip_ratio/high_max": 0.0,
@@ -7772,26 +7772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 168.0,
- "completions/max_terminated_length": 168.0,
- "completions/mean_length": 104.5,
- "completions/mean_terminated_length": 104.5,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5374161005020142,
- "epoch": 2.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.129844531416893,
- "kl": 3.9368313082377426e-05,
- "learning_rate": 1.4224118781404917e-07,
- "loss": -0.1482687145471573,
- "num_tokens": 1147731.0,
- "reward": 0.43388551473617554,
- "reward_std": 0.39525240659713745,
- "rewards/true_env_reward_fn/mean": 0.43388551473617554,
- "rewards/true_env_reward_fn/std": 0.39525243639945984,
+ "completions/max_length": 149.0,
+ "completions/max_terminated_length": 149.0,
+ "completions/mean_length": 71.60417175292969,
+ "completions/mean_terminated_length": 71.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2196767628192902,
+ "epoch": 6.780487804878049,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03178449347615242,
+ "kl": 5.627466816804372e-05,
+ "learning_rate": 2.8772683296076194e-07,
+ "loss": 0.002586783841252327,
+ "num_tokens": 6852617.0,
+ "reward": 0.5281299352645874,
+ "reward_std": 0.22268518805503845,
+ "rewards/true_env_reward_fn/mean": 0.5281298756599426,
+ "rewards/true_env_reward_fn/std": 0.22268518805503845,
"step": 278,
- "step_time": 7.55689369099855
+ "step_time": 11.629210506000163
},
{
"clip_ratio/high_max": 0.0,
@@ -7800,26 +7800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.0975646376609802,
- "epoch": 2.2682926829268295,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1335010528564453,
- "kl": 2.2682882445224095e-05,
- "learning_rate": 1.3935837587390214e-07,
- "loss": -0.009766265749931335,
- "num_tokens": 1151246.0,
- "reward": 0.37655720114707947,
- "reward_std": 0.37258440256118774,
- "rewards/true_env_reward_fn/mean": 0.37655720114707947,
- "rewards/true_env_reward_fn/std": 0.37258440256118774,
+ "completions/max_length": 169.0,
+ "completions/max_terminated_length": 169.0,
+ "completions/mean_length": 68.66667175292969,
+ "completions/mean_terminated_length": 68.66667175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1991542279720306,
+ "epoch": 6.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07567304372787476,
+ "kl": 0.00015181046910583973,
+ "learning_rate": 2.8388034850262646e-07,
+ "loss": -0.04906900227069855,
+ "num_tokens": 6881529.0,
+ "reward": 0.3906375765800476,
+ "reward_std": 0.3576822578907013,
+ "rewards/true_env_reward_fn/mean": 0.3906375467777252,
+ "rewards/true_env_reward_fn/std": 0.3576822876930237,
"step": 279,
- "step_time": 3.8792882219986495
+ "step_time": 18.253660386000774
},
{
"clip_ratio/high_max": 0.0,
@@ -7828,26 +7828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1846895217895508,
- "epoch": 2.2764227642276422,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.138702854514122,
- "kl": 2.0143882466072682e-05,
- "learning_rate": 1.3649890195928254e-07,
- "loss": 0.013748884201049805,
- "num_tokens": 1153222.0,
- "reward": 0.8218116760253906,
- "reward_std": 0.1293545663356781,
- "rewards/true_env_reward_fn/mean": 0.8218116760253906,
- "rewards/true_env_reward_fn/std": 0.1293545812368393,
+ "completions/max_length": 208.0,
+ "completions/max_terminated_length": 208.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.263102412223816,
+ "epoch": 6.829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09104589372873306,
+ "kl": 0.00012209633769089123,
+ "learning_rate": 2.8004952934203837e-07,
+ "loss": 0.06642289459705353,
+ "num_tokens": 6905111.0,
+ "reward": 0.3988339304924011,
+ "reward_std": 0.34396088123321533,
+ "rewards/true_env_reward_fn/mean": 0.39883390069007874,
+ "rewards/true_env_reward_fn/std": 0.3439609110355377,
"step": 280,
- "step_time": 3.42550413599929
+ "step_time": 15.519001798998943
},
{
"clip_ratio/high_max": 0.0,
@@ -7856,26 +7856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 64.0,
- "completions/mean_terminated_length": 64.0,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3451175689697266,
- "epoch": 2.2845528455284554,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1508527249097824,
- "kl": 1.6968931049632374e-05,
- "learning_rate": 1.336630221093991e-07,
- "loss": 0.04001428931951523,
- "num_tokens": 1157606.0,
- "reward": 0.4994586706161499,
- "reward_std": 0.10593737661838531,
- "rewards/true_env_reward_fn/mean": 0.4994586706161499,
- "rewards/true_env_reward_fn/std": 0.10593737661838531,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 59.0625,
+ "completions/mean_terminated_length": 59.0625,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.208267629146576,
+ "epoch": 6.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07465670257806778,
+ "kl": 7.14320785846212e-05,
+ "learning_rate": 2.762346531535246e-07,
+ "loss": 0.04124641790986061,
+ "num_tokens": 6926634.0,
+ "reward": 0.5870868563652039,
+ "reward_std": 0.28217118978500366,
+ "rewards/true_env_reward_fn/mean": 0.5870868563652039,
+ "rewards/true_env_reward_fn/std": 0.2821711599826813,
"step": 281,
- "step_time": 3.914840199000537
+ "step_time": 11.224198447001072
},
{
"clip_ratio/high_max": 0.0,
@@ -7884,26 +7884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2436452507972717,
- "epoch": 2.292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12380471080541611,
- "kl": 4.350653580331709e-05,
- "learning_rate": 1.3085099025083245e-07,
- "loss": -0.029160797595977783,
- "num_tokens": 1162012.0,
- "reward": 0.5066306591033936,
- "reward_std": 0.28914663195610046,
- "rewards/true_env_reward_fn/mean": 0.5066306591033936,
- "rewards/true_env_reward_fn/std": 0.28914666175842285,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.02083587646484,
+ "completions/mean_terminated_length": 72.02083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3142951428890228,
+ "epoch": 6.878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07364725321531296,
+ "kl": 0.00011016946336894762,
+ "learning_rate": 2.7243599645599574e-07,
+ "loss": 0.004198473412543535,
+ "num_tokens": 6960219.0,
+ "reward": 0.34401482343673706,
+ "reward_std": 0.24699951708316803,
+ "rewards/true_env_reward_fn/mean": 0.3440147936344147,
+ "rewards/true_env_reward_fn/std": 0.24699951708316803,
"step": 282,
- "step_time": 3.9093819319987233
+ "step_time": 15.210776117999558
},
{
"clip_ratio/high_max": 0.0,
@@ -7912,26 +7912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 59.375,
- "completions/mean_terminated_length": 59.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.10621577501297,
- "epoch": 2.3008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001239393459400162,
- "kl": 1.6069413049990544e-05,
- "learning_rate": 1.2806305817479771e-07,
- "loss": 8.174432082341809e-07,
- "num_tokens": 1165367.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 60.583335876464844,
+ "completions/mean_terminated_length": 60.583335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2013934552669525,
+ "epoch": 6.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07040710747241974,
+ "kl": 7.988750212462037e-05,
+ "learning_rate": 2.686538345927026e-07,
+ "loss": 0.06334929168224335,
+ "num_tokens": 6982879.0,
+ "reward": 0.5139331221580505,
+ "reward_std": 0.19786590337753296,
+ "rewards/true_env_reward_fn/mean": 0.5139331221580505,
+ "rewards/true_env_reward_fn/std": 0.19786591827869415,
"step": 283,
- "step_time": 3.8117841049988783
+ "step_time": 10.240000448000501
},
{
"clip_ratio/high_max": 0.0,
@@ -7940,26 +7940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 66.0,
- "completions/mean_terminated_length": 66.0,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.382466197013855,
- "epoch": 2.3089430894308944,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1446879506111145,
- "kl": 3.576014205464162e-05,
- "learning_rate": 1.2529947551459964e-07,
- "loss": -0.04731176793575287,
- "num_tokens": 1169243.0,
- "reward": 0.496622234582901,
- "reward_std": 0.17860308289527893,
- "rewards/true_env_reward_fn/mean": 0.496622234582901,
- "rewards/true_env_reward_fn/std": 0.17860306799411774,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2223551571369171,
+ "epoch": 6.926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07971613109111786,
+ "kl": 0.00016717875041649677,
+ "learning_rate": 2.64888441711279e-07,
+ "loss": 0.03706979751586914,
+ "num_tokens": 7003990.0,
+ "reward": 0.5301157832145691,
+ "reward_std": 0.2464885115623474,
+ "rewards/true_env_reward_fn/mean": 0.5301157832145691,
+ "rewards/true_env_reward_fn/std": 0.2464885115623474,
"step": 284,
- "step_time": 3.649135475998264
+ "step_time": 9.309556909001913
},
{
"clip_ratio/high_max": 0.0,
@@ -7968,26 +7968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 151.0,
- "completions/max_terminated_length": 151.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1946157813072205,
- "epoch": 2.317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10612626373767853,
- "kl": 2.0257402866263874e-05,
- "learning_rate": 1.2256048972327967e-07,
- "loss": -0.029511645436286926,
- "num_tokens": 1173594.0,
- "reward": 0.5235810279846191,
- "reward_std": 0.2810492217540741,
- "rewards/true_env_reward_fn/mean": 0.5235810279846191,
- "rewards/true_env_reward_fn/std": 0.2810492217540741,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.309690535068512,
+ "epoch": 6.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09541535377502441,
+ "kl": 0.00014337312313728034,
+ "learning_rate": 2.6114009074386844e-07,
+ "loss": -0.04744558781385422,
+ "num_tokens": 7027733.0,
+ "reward": 0.468932569026947,
+ "reward_std": 0.2396899163722992,
+ "rewards/true_env_reward_fn/mean": 0.46893253922462463,
+ "rewards/true_env_reward_fn/std": 0.2396899312734604,
"step": 285,
- "step_time": 6.7011265000001
+ "step_time": 8.91306197799986
},
{
"clip_ratio/high_max": 0.0,
@@ -7996,26 +7996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2879216074943542,
- "epoch": 2.3252032520325203,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12234477698802948,
- "kl": 1.4488860415440286e-05,
- "learning_rate": 1.1984634605145978e-07,
- "loss": -0.05451745539903641,
- "num_tokens": 1176384.0,
- "reward": 0.6496104001998901,
- "reward_std": 0.3368098735809326,
- "rewards/true_env_reward_fn/mean": 0.6496104001998901,
- "rewards/true_env_reward_fn/std": 0.3368098735809326,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1709823310375214,
+ "epoch": 6.975609756097561,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048187412321567535,
+ "kl": 9.718035107653122e-05,
+ "learning_rate": 2.5740905338734306e-07,
+ "loss": 0.010684527456760406,
+ "num_tokens": 7058137.0,
+ "reward": 0.44316989183425903,
+ "reward_std": 0.14355739951133728,
+ "rewards/true_env_reward_fn/mean": 0.44316986203193665,
+ "rewards/true_env_reward_fn/std": 0.14355739951133728,
"step": 286,
- "step_time": 4.18911992899848
+ "step_time": 9.546786461999545
},
{
"clip_ratio/high_max": 0.0,
@@ -8024,26 +8024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 105.0,
- "completions/max_terminated_length": 105.0,
- "completions/mean_length": 70.875,
- "completions/mean_terminated_length": 70.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0649144053459167,
- "epoch": 2.3333333333333335,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1467445194721222,
- "kl": 3.947542245441582e-05,
- "learning_rate": 1.17157287525381e-07,
- "loss": -0.029090911149978638,
- "num_tokens": 1179651.0,
- "reward": 0.4754716753959656,
- "reward_std": 0.2500669062137604,
- "rewards/true_env_reward_fn/mean": 0.4754716753959656,
- "rewards/true_env_reward_fn/std": 0.2500669062137604,
+ "completions/max_length": 458.0,
+ "completions/max_terminated_length": 458.0,
+ "completions/mean_length": 77.97917175292969,
+ "completions/mean_terminated_length": 77.97917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3698437809944153,
+ "epoch": 7.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05576475337147713,
+ "kl": 9.964485252567101e-05,
+ "learning_rate": 2.536956000836082e-07,
+ "loss": 0.08634226769208908,
+ "num_tokens": 7082840.0,
+ "reward": 0.5030691027641296,
+ "reward_std": 0.2275215983390808,
+ "rewards/true_env_reward_fn/mean": 0.5030691027641296,
+ "rewards/true_env_reward_fn/std": 0.22752158343791962,
"step": 287,
- "step_time": 4.5810332049986755
+ "step_time": 26.703723129000537
},
{
"clip_ratio/high_max": 0.0,
@@ -8052,26 +8052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 1.002779871225357,
- "epoch": 2.341463414634146,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12533478438854218,
- "kl": 2.3054610210238025e-05,
- "learning_rate": 1.1449355492514437e-07,
- "loss": -0.04500773549079895,
- "num_tokens": 1181547.0,
- "reward": 0.7334807515144348,
- "reward_std": 0.12557923793792725,
- "rewards/true_env_reward_fn/mean": 0.7334807515144348,
- "rewards/true_env_reward_fn/std": 0.12557923793792725,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2532718479633331,
+ "epoch": 7.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08579511940479279,
+ "kl": 9.319775290350663e-05,
+ "learning_rate": 2.500000000000001e-07,
+ "loss": -0.013113420456647873,
+ "num_tokens": 7098689.0,
+ "reward": 0.5958684682846069,
+ "reward_std": 0.1863170713186264,
+ "rewards/true_env_reward_fn/mean": 0.5958684682846069,
+ "rewards/true_env_reward_fn/std": 0.1863170564174652,
"step": 288,
- "step_time": 3.8661079009998502
+ "step_time": 7.618657231000725
},
{
"clip_ratio/high_max": 0.0,
@@ -8080,26 +8080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 294.0,
- "completions/max_terminated_length": 294.0,
- "completions/mean_length": 81.25,
- "completions/mean_terminated_length": 81.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2280530333518982,
- "epoch": 2.3495934959349594,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23682235181331635,
- "kl": 5.290110129863024e-05,
- "learning_rate": 1.1185538676315052e-07,
- "loss": 0.5538168549537659,
- "num_tokens": 1183973.0,
- "reward": 0.5785378217697144,
- "reward_std": 0.29215970635414124,
- "rewards/true_env_reward_fn/mean": 0.5785378217697144,
- "rewards/true_env_reward_fn/std": 0.2921597361564636,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.20833587646484,
+ "completions/mean_terminated_length": 68.20833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.1685407161712646,
+ "epoch": 7.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07076086848974228,
+ "kl": 7.435419411194744e-05,
+ "learning_rate": 2.4632252100977564e-07,
+ "loss": 0.010196422226727009,
+ "num_tokens": 7126387.0,
+ "reward": 0.47137731313705444,
+ "reward_std": 0.20232117176055908,
+ "rewards/true_env_reward_fn/mean": 0.47137728333473206,
+ "rewards/true_env_reward_fn/std": 0.20232117176055908,
"step": 289,
- "step_time": 11.744046860998424
+ "step_time": 10.741382757999418
},
{
"clip_ratio/high_max": 0.0,
@@ -8108,26 +8108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.264986515045166,
- "epoch": 2.3577235772357725,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001335782726528123,
- "kl": 1.7416054106433876e-05,
- "learning_rate": 1.0924301926274248e-07,
- "loss": 8.602528396295384e-07,
- "num_tokens": 1187311.0,
- "reward": 0.4899469017982483,
- "reward_std": 0.24049179255962372,
- "rewards/true_env_reward_fn/mean": 0.4899469017982483,
- "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2131870985031128,
+ "epoch": 7.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07972414791584015,
+ "kl": 0.00010809541072376305,
+ "learning_rate": 2.426634296726955e-07,
+ "loss": 0.07707639783620834,
+ "num_tokens": 7150947.0,
+ "reward": 0.4951697587966919,
+ "reward_std": 0.31705158948898315,
+ "rewards/true_env_reward_fn/mean": 0.4951697587966919,
+ "rewards/true_env_reward_fn/std": 0.31705158948898315,
"step": 290,
- "step_time": 4.003069795000556
+ "step_time": 11.908707627000695
},
{
"clip_ratio/high_max": 0.0,
@@ -8136,26 +8136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.222926139831543,
- "epoch": 2.3658536585365852,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.209110481198877e-05,
- "kl": 1.1452370472397888e-05,
- "learning_rate": 1.0665668633705572e-07,
- "loss": 5.764911747974111e-07,
- "num_tokens": 1191370.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 66.41667175292969,
+ "completions/mean_terminated_length": 66.41667175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2247934639453888,
+ "epoch": 7.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06208934634923935,
+ "kl": 8.92497146196547e-05,
+ "learning_rate": 2.390229912157033e-07,
+ "loss": 0.021278446540236473,
+ "num_tokens": 7174671.0,
+ "reward": 0.44533461332321167,
+ "reward_std": 0.20755091309547424,
+ "rewards/true_env_reward_fn/mean": 0.4453345835208893,
+ "rewards/true_env_reward_fn/std": 0.20755092799663544,
"step": 291,
- "step_time": 4.77749846199913
+ "step_time": 11.859711304000484
},
{
"clip_ratio/high_max": 0.0,
@@ -8164,26 +8164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.313057780265808,
- "epoch": 2.3739837398373984,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.671779556199908e-05,
- "kl": 1.1384066056052689e-05,
- "learning_rate": 1.0409661956807174e-07,
- "loss": 5.689008162335085e-07,
- "num_tokens": 1195851.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.25,
+ "completions/mean_terminated_length": 66.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3344177305698395,
+ "epoch": 7.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08249509334564209,
+ "kl": 0.00013599474277725676,
+ "learning_rate": 2.3540146951369966e-07,
+ "loss": 0.05746981129050255,
+ "num_tokens": 7202291.0,
+ "reward": 0.36803489923477173,
+ "reward_std": 0.22159868478775024,
+ "rewards/true_env_reward_fn/mean": 0.36803486943244934,
+ "rewards/true_env_reward_fn/std": 0.22159868478775024,
"step": 292,
- "step_time": 3.4948791150000034
+ "step_time": 9.794866193998132
},
{
"clip_ratio/high_max": 0.0,
@@ -8192,26 +8192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.3050158619880676,
- "epoch": 2.3821138211382116,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15694627165794373,
- "kl": 2.6891467314271722e-05,
- "learning_rate": 1.0156304818588308e-07,
- "loss": -0.044491954147815704,
- "num_tokens": 1201316.0,
- "reward": 0.18382371962070465,
- "reward_std": 0.18414245545864105,
- "rewards/true_env_reward_fn/mean": 0.18382371962070465,
- "rewards/true_env_reward_fn/std": 0.18414245545864105,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 64.02083587646484,
+ "completions/mean_terminated_length": 64.02083587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2605153024196625,
+ "epoch": 7.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07226194441318512,
+ "kl": 7.86567807153915e-05,
+ "learning_rate": 2.3179912707041666e-07,
+ "loss": -0.06701754778623581,
+ "num_tokens": 7221060.0,
+ "reward": 0.6259939670562744,
+ "reward_std": 0.30331701040267944,
+ "rewards/true_env_reward_fn/mean": 0.6259939074516296,
+ "rewards/true_env_reward_fn/std": 0.30331701040267944,
"step": 293,
- "step_time": 3.900356202000694
+ "step_time": 13.536596455999643
},
{
"clip_ratio/high_max": 0.0,
@@ -8220,26 +8220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 52.25,
- "completions/mean_terminated_length": 52.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2079132199287415,
- "epoch": 2.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25304585695266724,
- "kl": 4.250183792464668e-05,
- "learning_rate": 9.905619904816749e-08,
- "loss": 0.008140146732330322,
- "num_tokens": 1204090.0,
- "reward": 0.43626630306243896,
- "reward_std": 0.3027261197566986,
- "rewards/true_env_reward_fn/mean": 0.43626630306243896,
- "rewards/true_env_reward_fn/std": 0.3027261197566986,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 76.83333587646484,
+ "completions/mean_terminated_length": 76.83333587646484,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.275952398777008,
+ "epoch": 7.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07967559248209,
+ "kl": 0.0001376894815621199,
+ "learning_rate": 2.2821622499938948e-07,
+ "loss": -0.024285754188895226,
+ "num_tokens": 7251156.0,
+ "reward": 0.3353421688079834,
+ "reward_std": 0.26816248893737793,
+ "rewards/true_env_reward_fn/mean": 0.3353421688079834,
+ "rewards/true_env_reward_fn/std": 0.26816248893737793,
"step": 294,
- "step_time": 3.1466946830005327
+ "step_time": 19.762229363001097
},
{
"clip_ratio/high_max": 0.0,
@@ -8248,26 +8248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2324314713478088,
- "epoch": 2.3983739837398375,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011851672024931759,
- "kl": 1.4868088328512385e-05,
- "learning_rate": 9.657629661987531e-08,
- "loss": 7.534490578109398e-07,
- "num_tokens": 1206449.0,
- "reward": 0.7232838273048401,
- "reward_std": 0.008955853059887886,
- "rewards/true_env_reward_fn/mean": 0.7232838273048401,
- "rewards/true_env_reward_fn/std": 0.008955853059887886,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.4140338003635406,
+ "epoch": 7.195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07337357848882675,
+ "kl": 0.00012654263991862535,
+ "learning_rate": 2.2465302300503008e-07,
+ "loss": -0.04723845049738884,
+ "num_tokens": 7277361.0,
+ "reward": 0.4155184030532837,
+ "reward_std": 0.23990045487880707,
+ "rewards/true_env_reward_fn/mean": 0.4155184030532837,
+ "rewards/true_env_reward_fn/std": 0.23990046977996826,
"step": 295,
- "step_time": 3.5492840760016406
+ "step_time": 11.263231479000751
},
{
"clip_ratio/high_max": 0.0,
@@ -8276,26 +8276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 65.125,
- "completions/mean_terminated_length": 65.125,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.2110244631767273,
- "epoch": 2.40650406504065,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 63.9375,
+ "completions/mean_terminated_length": 63.9375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.22734934091568,
+ "epoch": 7.219512195121951,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.18513579666614532,
- "kl": 6.467000457632821e-05,
- "learning_rate": 9.412356295313019e-08,
- "loss": -0.008508354425430298,
- "num_tokens": 1211294.0,
- "reward": 0.38126999139785767,
- "reward_std": 0.22339044511318207,
- "rewards/true_env_reward_fn/mean": 0.38126999139785767,
- "rewards/true_env_reward_fn/std": 0.22339043021202087,
+ "grad_norm": 0.0908711850643158,
+ "kl": 0.00016623977535346057,
+ "learning_rate": 2.2110977936380287e-07,
+ "loss": -0.03445049002766609,
+ "num_tokens": 7298998.0,
+ "reward": 0.564518392086029,
+ "reward_std": 0.30392351746559143,
+ "rewards/true_env_reward_fn/mean": 0.564518392086029,
+ "rewards/true_env_reward_fn/std": 0.30392348766326904,
"step": 296,
- "step_time": 3.6129159619995335
+ "step_time": 10.9702629049998
},
{
"clip_ratio/high_max": 0.0,
@@ -8304,26 +8304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1828523874282837,
- "epoch": 2.4146341463414633,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13915051519870758,
- "kl": 2.76857699645916e-05,
- "learning_rate": 9.169821766734668e-08,
- "loss": -0.019797056913375854,
- "num_tokens": 1214972.0,
- "reward": 0.5778937339782715,
- "reward_std": 0.4268997013568878,
- "rewards/true_env_reward_fn/mean": 0.5778937339782715,
- "rewards/true_env_reward_fn/std": 0.4268997013568878,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.312496393918991,
+ "epoch": 7.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08918203413486481,
+ "kl": 0.00018221777827420738,
+ "learning_rate": 2.1758675090550328e-07,
+ "loss": 0.023428799584507942,
+ "num_tokens": 7320725.0,
+ "reward": 0.4695018231868744,
+ "reward_std": 0.2527172565460205,
+ "rewards/true_env_reward_fn/mean": 0.4695018231868744,
+ "rewards/true_env_reward_fn/std": 0.2527172565460205,
"step": 297,
- "step_time": 3.5299333029997797
+ "step_time": 12.689384352000161
},
{
"clip_ratio/high_max": 0.0,
@@ -8332,26 +8332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2796242237091064,
- "epoch": 2.4227642276422765,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23225097358226776,
- "kl": 5.458398300106637e-05,
- "learning_rate": 8.930047792956585e-08,
- "loss": 0.028596192598342896,
- "num_tokens": 1221117.0,
- "reward": 0.09695600718259811,
- "reward_std": 0.23755072057247162,
- "rewards/true_env_reward_fn/mean": 0.09695600718259811,
- "rewards/true_env_reward_fn/std": 0.23755072057247162,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.260593295097351,
+ "epoch": 7.2682926829268295,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07724963873624802,
+ "kl": 4.775456409333856e-05,
+ "learning_rate": 2.1408419299464242e-07,
+ "loss": 0.03472111374139786,
+ "num_tokens": 7345397.0,
+ "reward": 0.39238378405570984,
+ "reward_std": 0.23648974299430847,
+ "rewards/true_env_reward_fn/mean": 0.39238378405570984,
+ "rewards/true_env_reward_fn/std": 0.23648977279663086,
"step": 298,
- "step_time": 3.9654863289997593
+ "step_time": 10.398283558999537
},
{
"clip_ratio/high_max": 0.0,
@@ -8360,26 +8360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3131609559059143,
- "epoch": 2.430894308943089,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15491950511932373,
- "kl": 2.3678386241954286e-05,
- "learning_rate": 8.693055843500867e-08,
- "loss": 0.08264091610908508,
- "num_tokens": 1226670.0,
- "reward": 0.22746901214122772,
- "reward_std": 0.2765822112560272,
- "rewards/true_env_reward_fn/mean": 0.22746901214122772,
- "rewards/true_env_reward_fn/std": 0.2765822112560272,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.272550791501999,
+ "epoch": 7.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0737687349319458,
+ "kl": 7.711273838140187e-05,
+ "learning_rate": 2.1060235951193578e-07,
+ "loss": 0.010874807834625244,
+ "num_tokens": 7371231.0,
+ "reward": 0.4486481845378876,
+ "reward_std": 0.26590272784233093,
+ "rewards/true_env_reward_fn/mean": 0.4486481845378876,
+ "rewards/true_env_reward_fn/std": 0.26590269804000854,
"step": 299,
- "step_time": 3.9300464680000005
+ "step_time": 12.961759718001304
},
{
"clip_ratio/high_max": 0.0,
@@ -8388,32 +8388,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 61.75,
- "completions/mean_terminated_length": 61.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.113481342792511,
- "epoch": 2.4390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14261895418167114,
- "kl": 2.8488028874562588e-05,
- "learning_rate": 8.458867138785369e-08,
- "loss": 0.012870386242866516,
- "num_tokens": 1230460.0,
- "reward": 0.6178936958312988,
- "reward_std": 0.3830615282058716,
- "rewards/true_env_reward_fn/mean": 0.6178936958312988,
- "rewards/true_env_reward_fn/std": 0.3830614984035492,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3426357805728912,
+ "epoch": 7.317073170731708,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09169661998748779,
+ "kl": 0.00015296797573682852,
+ "learning_rate": 2.071415028359026e-07,
+ "loss": -0.0708877444267273,
+ "num_tokens": 7397151.0,
+ "reward": 0.5012298822402954,
+ "reward_std": 0.21363919973373413,
+ "rewards/true_env_reward_fn/mean": 0.5012298226356506,
+ "rewards/true_env_reward_fn/std": 0.21363921463489532,
"step": 300,
- "step_time": 4.004705740000645
+ "step_time": 13.761256955000135
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 1230460,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 7397151,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -8428,7 +8428,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-300/training_args.bin b/checkpoint-300/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-300/training_args.bin
+++ b/checkpoint-300/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-350/adapter_model.safetensors b/checkpoint-350/adapter_model.safetensors
index 6c75e65da0669d9eb4342a5c1b4417e5b41afa9c..57aad585606e0fb32839fb305e02f95f758c471c 100644
--- a/checkpoint-350/adapter_model.safetensors
+++ b/checkpoint-350/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:533eaa168befd22710329923069f2ecd67a71f79884c32b38445407693b00e42
+oid sha256:d943b92e5105877266ac6c9766c5799f73fa7969bfeb3bcc4efd114b91ca136a
size 8731128
diff --git a/checkpoint-350/optimizer.pt b/checkpoint-350/optimizer.pt
index af8766d559bb5cb2ea3b62cd3247d6e28407e714..03af1cfeaebae54afc3ea6338ed42eef27009d10 100644
--- a/checkpoint-350/optimizer.pt
+++ b/checkpoint-350/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:1a18e246c986121a8a202c83b1f88c188e77479d9bb250438f23fb7a9a8cc5e8
+oid sha256:4e2f5f20308f78f86c9fa92754f04090dbd133b411e046d4cc8fdb0b1e55c6f8
size 17526842
diff --git a/checkpoint-350/ref/adapter_model.safetensors b/checkpoint-350/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-350/ref/adapter_model.safetensors
+++ b/checkpoint-350/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-350/rng_state.pth b/checkpoint-350/rng_state.pth
index 4e9803d1d6b0a597a97c9cb707bb958f817e862b..a38be130378aecb520fcfb89bbd045ad743c93fc 100644
--- a/checkpoint-350/rng_state.pth
+++ b/checkpoint-350/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d57f3e3ca070293c7549a6259323a36e5610ff41b3a9d1573749f4e44986530c
+oid sha256:15f235073cda7f342278db8cde4ba70662f98e9016ead5bfcf229cc1f8372f57
size 14244
diff --git a/checkpoint-350/scheduler.pt b/checkpoint-350/scheduler.pt
index 935bf34e5e31b5db56921a8cf308426c15663715..bc0f373d96ce72a0f799c8871eb412237739e35b 100644
--- a/checkpoint-350/scheduler.pt
+++ b/checkpoint-350/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:e41de323bdf0c331883c417e3b3eefaa749e3ae7420b215532e342bfd966244c
+oid sha256:824b8149acabea0391619307b03f066171991bb80500311ead9736976d3ec2c0
size 1064
diff --git a/checkpoint-350/trainer_state.json b/checkpoint-350/trainer_state.json
index d80a7247e12d58560ecba3c73a502159f5695ce8..194fba3901756973a87fc61f0da10eaa961c96dc 100644
--- a/checkpoint-350/trainer_state.json
+++ b/checkpoint-350/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 2.845528455284553,
+ "epoch": 8.536585365853659,
"eval_steps": 500,
"global_step": 350,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,26 +1388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
},
{
"clip_ratio/high_max": 0.0,
@@ -1416,26 +1416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 0.9613964557647705,
- "epoch": 0.4146341463414634,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12812700867652893,
- "kl": 1.0464088063599775e-05,
- "learning_rate": 7.969773173984153e-07,
- "loss": 0.023206032812595367,
- "num_tokens": 210443.0,
- "reward": 0.3208000063896179,
- "reward_std": 0.25050169229507446,
- "rewards/true_env_reward_fn/mean": 0.3208000063896179,
- "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
"step": 51,
- "step_time": 3.6275602460009395
+ "step_time": 10.88732858200001
},
{
"clip_ratio/high_max": 0.0,
@@ -1444,26 +1444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2359730005264282,
- "epoch": 0.42276422764227645,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1384950578212738,
- "kl": 1.2094554222130682e-05,
- "learning_rate": 7.964951099967749e-07,
- "loss": -0.07054222375154495,
- "num_tokens": 213833.0,
- "reward": 0.5900156497955322,
- "reward_std": 0.18237514793872833,
- "rewards/true_env_reward_fn/mean": 0.5900156497955322,
- "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
"step": 52,
- "step_time": 3.8849526029989647
+ "step_time": 9.252596976000177
},
{
"clip_ratio/high_max": 0.0,
@@ -1472,26 +1472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 79.5,
- "completions/mean_terminated_length": 79.5,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2706108689308167,
- "epoch": 0.43089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17388403415679932,
- "kl": 1.3583369309344562e-05,
- "learning_rate": 7.959774001575264e-07,
- "loss": 0.06114684417843819,
- "num_tokens": 216853.0,
- "reward": 0.4848448634147644,
- "reward_std": 0.2859330177307129,
- "rewards/true_env_reward_fn/mean": 0.4848448634147644,
- "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
"step": 53,
- "step_time": 4.964324356000361
+ "step_time": 24.22518693999973
},
{
"clip_ratio/high_max": 0.0,
@@ -1500,26 +1500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2430712580680847,
- "epoch": 0.43902439024390244,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11113106459379196,
- "kl": 1.2204414360894589e-05,
- "learning_rate": 7.954242342367553e-07,
- "loss": 0.010590985417366028,
- "num_tokens": 221252.0,
- "reward": 0.392258882522583,
- "reward_std": 0.13280020654201508,
- "rewards/true_env_reward_fn/mean": 0.392258882522583,
- "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
"step": 54,
- "step_time": 3.5511989209990134
+ "step_time": 10.14821418500037
},
{
"clip_ratio/high_max": 0.0,
@@ -1528,26 +1528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3175880908966064,
- "epoch": 0.44715447154471544,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20881597697734833,
- "kl": 1.58558846123924e-05,
- "learning_rate": 7.948356617653087e-07,
- "loss": -0.06772151589393616,
- "num_tokens": 224691.0,
- "reward": 0.30961817502975464,
- "reward_std": 0.27422165870666504,
- "rewards/true_env_reward_fn/mean": 0.30961817502975464,
- "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
"step": 55,
- "step_time": 5.031640098000935
+ "step_time": 11.13082981000025
},
{
"clip_ratio/high_max": 0.0,
@@ -1556,26 +1556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 64.625,
- "completions/mean_terminated_length": 64.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.4056915640830994,
- "epoch": 0.45528455284552843,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.606108895037323e-05,
- "kl": 1.2847603557020193e-05,
- "learning_rate": 7.942117354443597e-07,
- "loss": 6.408997705875663e-07,
- "num_tokens": 228116.0,
- "reward": 0.5599601864814758,
- "reward_std": 0.18355616927146912,
- "rewards/true_env_reward_fn/mean": 0.5599601864814758,
- "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
"step": 56,
- "step_time": 3.6221305880008003
+ "step_time": 14.49393488900023
},
{
"clip_ratio/high_max": 0.0,
@@ -1584,26 +1584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.4034882187843323,
- "epoch": 0.4634146341463415,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19157616794109344,
- "kl": 1.4551038475474343e-05,
- "learning_rate": 7.935525111406885e-07,
- "loss": 0.021202675998210907,
- "num_tokens": 233139.0,
- "reward": 0.32785865664482117,
- "reward_std": 0.2835054397583008,
- "rewards/true_env_reward_fn/mean": 0.32785865664482117,
- "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
"step": 57,
- "step_time": 3.7005361410010664
+ "step_time": 14.065935286000013
},
{
"clip_ratio/high_max": 0.0,
@@ -1612,26 +1612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 173.0,
- "completions/max_terminated_length": 173.0,
- "completions/mean_length": 76.625,
- "completions/mean_terminated_length": 76.625,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2941595911979675,
- "epoch": 0.4715447154471545,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14299193024635315,
- "kl": 1.3164159554435173e-05,
- "learning_rate": 7.92858047881681e-07,
- "loss": -0.14726585149765015,
- "num_tokens": 238584.0,
- "reward": 0.444433331489563,
- "reward_std": 0.030650291591882706,
- "rewards/true_env_reward_fn/mean": 0.444433331489563,
- "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
"step": 58,
- "step_time": 7.550715425000817
+ "step_time": 12.76291504100027
},
{
"clip_ratio/high_max": 0.0,
@@ -1640,26 +1640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1917714476585388,
- "epoch": 0.4796747967479675,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25083038210868835,
- "kl": 1.3176229913369752e-05,
- "learning_rate": 7.921284078500422e-07,
- "loss": 0.088463693857193,
- "num_tokens": 240669.0,
- "reward": 0.7982887029647827,
- "reward_std": 0.1672983169555664,
- "rewards/true_env_reward_fn/mean": 0.7982887029647827,
- "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
"step": 59,
- "step_time": 3.7769912429994292
+ "step_time": 11.089177285999767
},
{
"clip_ratio/high_max": 0.0,
@@ -1668,26 +1668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.375,
- "completions/mean_terminated_length": 66.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3743653893470764,
- "epoch": 0.4878048780487805,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18919643759727478,
- "kl": 1.231462101713987e-05,
- "learning_rate": 7.91363656378229e-07,
- "loss": -0.08548973500728607,
- "num_tokens": 243808.0,
- "reward": 0.5988538861274719,
- "reward_std": 0.11870570480823517,
- "rewards/true_env_reward_fn/mean": 0.5988538861274719,
- "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
"step": 60,
- "step_time": 4.052767743998629
+ "step_time": 12.328215124000053
},
{
"clip_ratio/high_max": 0.0,
@@ -1696,26 +1696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 77.125,
- "completions/mean_terminated_length": 77.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.329764723777771,
- "epoch": 0.4959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1690233051776886,
- "kl": 1.405783814334427e-05,
- "learning_rate": 7.905638619426003e-07,
- "loss": 0.0050433604046702385,
- "num_tokens": 248725.0,
- "reward": 0.27516257762908936,
- "reward_std": 0.32322537899017334,
- "rewards/true_env_reward_fn/mean": 0.27516257762908936,
- "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
"step": 61,
- "step_time": 6.010593229999358
+ "step_time": 13.294195681999781
},
{
"clip_ratio/high_max": 0.0,
@@ -1724,26 +1724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.2542970776557922,
- "epoch": 0.5040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11462891101837158,
- "kl": 1.13775058707688e-05,
- "learning_rate": 7.897290961572853e-07,
- "loss": -0.007184989750385284,
- "num_tokens": 252101.0,
- "reward": 0.5372380018234253,
- "reward_std": 0.13500821590423584,
- "rewards/true_env_reward_fn/mean": 0.5372380018234253,
- "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
"step": 62,
- "step_time": 3.4512634010006877
+ "step_time": 18.972790307999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1752,26 +1752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 65.75,
- "completions/mean_terminated_length": 65.75,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1982964873313904,
- "epoch": 0.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12309258431196213,
- "kl": 1.69004347299051e-05,
- "learning_rate": 7.888594337677712e-07,
- "loss": 0.0009508281946182251,
- "num_tokens": 255231.0,
- "reward": 0.6114543080329895,
- "reward_std": 0.10413603484630585,
- "rewards/true_env_reward_fn/mean": 0.6114543080329895,
- "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
"step": 63,
- "step_time": 3.735559521997857
+ "step_time": 11.292631820999986
},
{
"clip_ratio/high_max": 0.0,
@@ -1780,26 +1780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 69.0,
- "completions/mean_terminated_length": 69.0,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3587612509727478,
- "epoch": 0.5203252032520326,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15858450531959534,
- "kl": 1.4598341294913553e-05,
- "learning_rate": 7.879549526442108e-07,
- "loss": 0.0696716383099556,
- "num_tokens": 260523.0,
- "reward": 0.2912999987602234,
- "reward_std": 0.2844822406768799,
- "rewards/true_env_reward_fn/mean": 0.2912999987602234,
- "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
"step": 64,
- "step_time": 5.731267729999672
+ "step_time": 9.600786530999585
},
{
"clip_ratio/high_max": 0.0,
@@ -1808,26 +1808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 230.0,
- "completions/max_terminated_length": 230.0,
- "completions/mean_length": 83.5,
- "completions/mean_terminated_length": 83.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2284430861473083,
- "epoch": 0.5284552845528455,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13374178111553192,
- "kl": 1.2341822639427846e-05,
- "learning_rate": 7.870157337744494e-07,
- "loss": 0.10693901032209396,
- "num_tokens": 264967.0,
- "reward": 0.3284733295440674,
- "reward_std": 0.3848404288291931,
- "rewards/true_env_reward_fn/mean": 0.3284733295440674,
- "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
"step": 65,
- "step_time": 9.601442954000959
+ "step_time": 10.169144185999812
},
{
"clip_ratio/high_max": 0.0,
@@ -1836,26 +1836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2396279573440552,
- "epoch": 0.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08427461981773376,
- "kl": 1.4658115105703473e-05,
- "learning_rate": 7.860418612567733e-07,
- "loss": -0.05642998591065407,
- "num_tokens": 269717.0,
- "reward": 0.38946664333343506,
- "reward_std": 0.1897086799144745,
- "rewards/true_env_reward_fn/mean": 0.38946664333343506,
- "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
"step": 66,
- "step_time": 6.017849919000582
+ "step_time": 10.343706631000032
},
{
"clip_ratio/high_max": 0.0,
@@ -1864,26 +1864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.2245049476623535,
- "epoch": 0.5447154471544715,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13357418775558472,
- "kl": 1.2806529412046075e-05,
- "learning_rate": 7.850334222923798e-07,
- "loss": 0.03744228184223175,
- "num_tokens": 275407.0,
- "reward": 0.08966667205095291,
- "reward_std": 0.23612774908542633,
- "rewards/true_env_reward_fn/mean": 0.08966667205095291,
- "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
"step": 67,
- "step_time": 4.4363536659984675
+ "step_time": 12.16719347000003
},
{
"clip_ratio/high_max": 0.0,
@@ -1892,26 +1892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2679882645606995,
- "epoch": 0.5528455284552846,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15667268633842468,
- "kl": 1.2213955869810889e-05,
- "learning_rate": 7.83990507177569e-07,
- "loss": -0.052396662533283234,
- "num_tokens": 280838.0,
- "reward": 0.2431039959192276,
- "reward_std": 0.2672288715839386,
- "rewards/true_env_reward_fn/mean": 0.2431039959192276,
- "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
"step": 68,
- "step_time": 3.6370441849994677
+ "step_time": 15.523659553000016
},
{
"clip_ratio/high_max": 0.0,
@@ -1920,26 +1920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 62.625,
- "completions/mean_terminated_length": 62.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2563416361808777,
- "epoch": 0.5609756097560976,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.937557868193835e-05,
- "kl": 1.1138304216729011e-05,
- "learning_rate": 7.829132092956586e-07,
- "loss": 5.569941095018294e-07,
- "num_tokens": 283603.0,
- "reward": 0.6040733456611633,
- "reward_std": 0.0834638923406601,
- "rewards/true_env_reward_fn/mean": 0.6040733456611633,
- "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
"step": 69,
- "step_time": 3.466609713001162
+ "step_time": 9.677535032999685
},
{
"clip_ratio/high_max": 0.0,
@@ -1948,26 +1948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 126.0,
- "completions/mean_terminated_length": 126.0,
- "completions/min_length": 76.0,
- "completions/min_terminated_length": 76.0,
- "entropy": 1.8668264746665955,
- "epoch": 0.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11100972443819046,
- "kl": 1.3833193406753708e-05,
- "learning_rate": 7.81801625108622e-07,
- "loss": -0.04258224368095398,
- "num_tokens": 290511.0,
- "reward": 0.37345871329307556,
- "reward_std": 0.016035744920372963,
- "rewards/true_env_reward_fn/mean": 0.37345871329307556,
- "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
"step": 70,
- "step_time": 8.357124549000218
+ "step_time": 15.20990351499995
},
{
"clip_ratio/high_max": 0.0,
@@ -1976,26 +1976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.1647167801856995,
- "epoch": 0.5772357723577236,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.12842044234275818,
- "kl": 1.35402724481537e-05,
- "learning_rate": 7.806558541484517e-07,
- "loss": -0.0010651163756847382,
- "num_tokens": 294315.0,
- "reward": 0.6432806849479675,
- "reward_std": 0.2300010770559311,
- "rewards/true_env_reward_fn/mean": 0.6432806849479675,
- "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
"step": 71,
- "step_time": 3.8402047919989855
+ "step_time": 14.703903473999844
},
{
"clip_ratio/high_max": 0.0,
@@ -2004,26 +2004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1465299725532532,
- "epoch": 0.5853658536585366,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23560228943824768,
- "kl": 1.4576367902918719e-05,
- "learning_rate": 7.794759990082466e-07,
- "loss": -0.11232151836156845,
- "num_tokens": 297803.0,
- "reward": 0.30700522661209106,
- "reward_std": 0.3690750300884247,
- "rewards/true_env_reward_fn/mean": 0.30700522661209106,
- "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
"step": 72,
- "step_time": 3.467162693001228
+ "step_time": 9.422353613000269
},
{
"clip_ratio/high_max": 0.0,
@@ -2032,26 +2032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.2479569911956787,
- "epoch": 0.5934959349593496,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011426099081290886,
- "kl": 1.304310217165039e-05,
- "learning_rate": 7.782621653330256e-07,
- "loss": 6.391838383024151e-07,
- "num_tokens": 301427.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
"step": 73,
- "step_time": 5.824168748999
+ "step_time": 16.65922043799992
},
{
"clip_ratio/high_max": 0.0,
@@ -2060,26 +2060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.170280933380127,
- "epoch": 0.6016260162601627,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22593456506729126,
- "kl": 2.0052431864314713e-05,
- "learning_rate": 7.77014461810269e-07,
- "loss": 0.16111303865909576,
- "num_tokens": 305492.0,
- "reward": 0.3909183144569397,
- "reward_std": 0.21756574511528015,
- "rewards/true_env_reward_fn/mean": 0.3909183144569397,
- "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
"step": 74,
- "step_time": 4.510902927002462
+ "step_time": 13.422026366999944
},
{
"clip_ratio/high_max": 0.0,
@@ -2088,26 +2088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2373355031013489,
- "epoch": 0.6097560975609756,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.604005466215312e-05,
- "kl": 1.0138399375136942e-05,
- "learning_rate": 7.757330001601855e-07,
- "loss": 5.069200028628984e-07,
- "num_tokens": 309826.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
"step": 75,
- "step_time": 3.6695911980004894
+ "step_time": 9.51117546200021
},
{
"clip_ratio/high_max": 0.0,
@@ -2116,26 +2116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.1873346865177155,
- "epoch": 0.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2067025899887085,
- "kl": 1.4842833934380906e-05,
- "learning_rate": 7.744178951257091e-07,
- "loss": -0.036428727209568024,
- "num_tokens": 316885.0,
- "reward": 0.13499999046325684,
- "reward_std": 0.23260429501533508,
- "rewards/true_env_reward_fn/mean": 0.13499999046325684,
- "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
"step": 76,
- "step_time": 4.359561059001862
+ "step_time": 9.295928349999713
},
{
"clip_ratio/high_max": 0.0,
@@ -2144,26 +2144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0709484219551086,
- "epoch": 0.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18151503801345825,
- "kl": 1.3910183042753488e-05,
- "learning_rate": 7.730692644622251e-07,
- "loss": -0.06179043650627136,
- "num_tokens": 319230.0,
- "reward": 0.6732838153839111,
- "reward_std": 0.1450435221195221,
- "rewards/true_env_reward_fn/mean": 0.6732838153839111,
- "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
"step": 77,
- "step_time": 3.1786108079995756
+ "step_time": 15.693113021000045
},
{
"clip_ratio/high_max": 0.0,
@@ -2172,26 +2172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 72.375,
- "completions/mean_terminated_length": 72.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5439093112945557,
- "epoch": 0.6341463414634146,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20670665800571442,
- "kl": 1.7317805031780154e-05,
- "learning_rate": 7.716872289270261e-07,
- "loss": -0.0654018223285675,
- "num_tokens": 324633.0,
- "reward": 0.23838475346565247,
- "reward_std": 0.2594907879829407,
- "rewards/true_env_reward_fn/mean": 0.23838475346565247,
- "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
"step": 78,
- "step_time": 4.930556027000421
+ "step_time": 16.146651725000083
},
{
"clip_ratio/high_max": 0.0,
@@ -2200,26 +2200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 47.625,
- "completions/mean_terminated_length": 47.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1996066868305206,
- "epoch": 0.6422764227642277,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21137002110481262,
- "kl": 1.325221819570288e-05,
- "learning_rate": 7.702719122684991e-07,
- "loss": 0.003889208659529686,
- "num_tokens": 329142.0,
- "reward": 0.3934500217437744,
- "reward_std": 0.1389254629611969,
- "rewards/true_env_reward_fn/mean": 0.3934500217437744,
- "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
"step": 79,
- "step_time": 3.5688320999997813
+ "step_time": 9.82867347299998
},
{
"clip_ratio/high_max": 0.0,
@@ -2228,26 +2228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4094278812408447,
- "epoch": 0.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17559278011322021,
- "kl": 1.6261046312138205e-05,
- "learning_rate": 7.688234412150453e-07,
- "loss": -0.04887707903981209,
- "num_tokens": 331663.0,
- "reward": 0.49859046936035156,
- "reward_std": 0.12171231955289841,
- "rewards/true_env_reward_fn/mean": 0.49859046936035156,
- "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
"step": 80,
- "step_time": 3.7867210379990865
+ "step_time": 11.267316974000096
},
{
"clip_ratio/high_max": 0.0,
@@ -2256,26 +2256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1693094372749329,
- "epoch": 0.6585365853658537,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010281114373356104,
- "kl": 1.2930718639836414e-05,
- "learning_rate": 7.673419454637328e-07,
- "loss": 6.465359092544531e-07,
- "num_tokens": 334637.0,
- "reward": 0.5707399845123291,
- "reward_std": 0.11909874528646469,
- "rewards/true_env_reward_fn/mean": 0.5707399845123291,
- "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
"step": 81,
- "step_time": 3.4751437539998733
+ "step_time": 10.035370067999793
},
{
"clip_ratio/high_max": 0.0,
@@ -2284,26 +2284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3214005827903748,
- "epoch": 0.6666666666666666,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2361973226070404,
- "kl": 1.4227861356630456e-05,
- "learning_rate": 7.658275576686829e-07,
- "loss": -0.08402466773986816,
- "num_tokens": 341701.0,
- "reward": 0.09331665933132172,
- "reward_std": 0.2172754853963852,
- "rewards/true_env_reward_fn/mean": 0.09331665933132172,
- "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
"step": 82,
- "step_time": 4.433740980000948
+ "step_time": 9.56242024800008
},
{
"clip_ratio/high_max": 0.0,
@@ -2311,27 +2311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 142.375,
- "completions/mean_terminated_length": 89.5714340209961,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.817092776298523,
- "epoch": 0.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.11815346032381058,
- "kl": 1.6899173715501092e-05,
- "learning_rate": 7.642804134291927e-07,
- "loss": -0.09939523041248322,
- "num_tokens": 346380.0,
- "reward": 0.47429025173187256,
- "reward_std": 0.24831563234329224,
- "rewards/true_env_reward_fn/mean": 0.47429025173187256,
- "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
"step": 83,
- "step_time": 20.738665008999305
+ "step_time": 10.563381390999666
},
{
"clip_ratio/high_max": 0.0,
@@ -2340,26 +2340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2211430668830872,
- "epoch": 0.6829268292682927,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20952872931957245,
- "kl": 1.2894654446427012e-05,
- "learning_rate": 7.62700651277593e-07,
- "loss": -0.0016747117042541504,
- "num_tokens": 351186.0,
- "reward": 0.386501669883728,
- "reward_std": 0.17392057180404663,
- "rewards/true_env_reward_fn/mean": 0.386501669883728,
- "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
"step": 84,
- "step_time": 4.028964023000299
+ "step_time": 17.21621736799966
},
{
"clip_ratio/high_max": 0.0,
@@ -2368,26 +2368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4367225170135498,
- "epoch": 0.6910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18939745426177979,
- "kl": 1.6035403859859798e-05,
- "learning_rate": 7.610884126668449e-07,
- "loss": 0.0628451332449913,
- "num_tokens": 355999.0,
- "reward": 0.5092726349830627,
- "reward_std": 0.2734805643558502,
- "rewards/true_env_reward_fn/mean": 0.5092726349830627,
- "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
"step": 85,
- "step_time": 4.244558566999331
+ "step_time": 25.519813745999954
},
{
"clip_ratio/high_max": 0.0,
@@ -2396,26 +2396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.25,
- "completions/mean_terminated_length": 44.25,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.009476900100708,
- "epoch": 0.6991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22703228890895844,
- "kl": 1.2845549463236239e-05,
- "learning_rate": 7.594438419578729e-07,
- "loss": -0.005728684365749359,
- "num_tokens": 360925.0,
- "reward": 0.28028765320777893,
- "reward_std": 0.2404259443283081,
- "rewards/true_env_reward_fn/mean": 0.28028765320777893,
- "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
"step": 86,
- "step_time": 2.618181756000922
+ "step_time": 10.78625990699993
},
{
"clip_ratio/high_max": 0.0,
@@ -2424,26 +2424,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.1686812043190002,
- "epoch": 0.7073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.401248098583892e-05,
- "kl": 1.2304412848607171e-05,
- "learning_rate": 7.577670864066391e-07,
- "loss": 6.143833388705389e-07,
- "num_tokens": 362399.0,
- "reward": 0.768503725528717,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.768503725528717,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
"step": 87,
- "step_time": 3.34067542199773
+ "step_time": 13.604215705999877
},
{
"clip_ratio/high_max": 0.0,
@@ -2451,27 +2451,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0876938998699188,
- "epoch": 0.7154471544715447,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010135328921023756,
- "kl": 1.3493038295564475e-05,
- "learning_rate": 7.560582961509586e-07,
- "loss": 6.750068450855906e-07,
- "num_tokens": 365500.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
"step": 88,
- "step_time": 3.3087227100004384
+ "step_time": 32.408574500999975
},
{
"clip_ratio/high_max": 0.0,
@@ -2480,26 +2480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.0288619995117188,
- "epoch": 0.7235772357723578,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010261479474138469,
- "kl": 1.3740621852775803e-05,
- "learning_rate": 7.543176241970547e-07,
- "loss": 6.875395683891838e-07,
- "num_tokens": 369222.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
"step": 89,
- "step_time": 3.786183243999403
+ "step_time": 9.901715897000031
},
{
"clip_ratio/high_max": 0.0,
@@ -2508,26 +2508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1757304668426514,
- "epoch": 0.7317073170731707,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2307090163230896,
- "kl": 2.1445125639729667e-05,
- "learning_rate": 7.525452264058595e-07,
- "loss": 0.12042637169361115,
- "num_tokens": 373465.0,
- "reward": 0.4571714401245117,
- "reward_std": 0.39374110102653503,
- "rewards/true_env_reward_fn/mean": 0.4571714401245117,
- "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
"step": 90,
- "step_time": 3.9787140030002774
+ "step_time": 11.152492722000034
},
{
"clip_ratio/high_max": 0.0,
@@ -2536,26 +2536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.302090346813202,
- "epoch": 0.7398373983739838,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16624286770820618,
- "kl": 1.6463789506815374e-05,
- "learning_rate": 7.507412614790579e-07,
- "loss": -0.05975423753261566,
- "num_tokens": 378029.0,
- "reward": 0.3388232886791229,
- "reward_std": 0.2467346489429474,
- "rewards/true_env_reward_fn/mean": 0.3388232886791229,
- "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
"step": 91,
- "step_time": 3.9565000490001694
+ "step_time": 10.656350811000038
},
{
"clip_ratio/high_max": 0.0,
@@ -2564,26 +2564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 77.75,
- "completions/mean_terminated_length": 77.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2768036723136902,
- "epoch": 0.7479674796747967,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10557293146848679,
- "kl": 1.2602345123013947e-05,
- "learning_rate": 7.489058909448776e-07,
- "loss": -0.023296140134334564,
- "num_tokens": 380883.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
"step": 92,
- "step_time": 4.720347813999979
+ "step_time": 10.304143018000104
},
{
"clip_ratio/high_max": 0.0,
@@ -2592,26 +2592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2670618891716003,
- "epoch": 0.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14662617444992065,
- "kl": 1.812677373891347e-05,
- "learning_rate": 7.470392791436244e-07,
- "loss": -0.05785401538014412,
- "num_tokens": 386095.0,
- "reward": 0.30487915873527527,
- "reward_std": 0.24597851932048798,
- "rewards/true_env_reward_fn/mean": 0.30487915873527527,
- "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
"step": 93,
- "step_time": 3.1318131530006212
+ "step_time": 13.589426085000014
},
{
"clip_ratio/high_max": 0.0,
@@ -2620,26 +2620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1208478510379791,
- "epoch": 0.7642276422764228,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011506211740197614,
- "kl": 1.2571507795655634e-05,
- "learning_rate": 7.451415932129691e-07,
- "loss": 6.294373520177032e-07,
- "num_tokens": 388335.0,
- "reward": 0.7244763970375061,
- "reward_std": 0.23028412461280823,
- "rewards/true_env_reward_fn/mean": 0.7244763970375061,
- "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
"step": 94,
- "step_time": 3.6959203189999243
+ "step_time": 14.587356482000132
},
{
"clip_ratio/high_max": 0.0,
@@ -2648,26 +2648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1998488903045654,
- "epoch": 0.7723577235772358,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12171207368373871,
- "kl": 1.6534771020815242e-05,
- "learning_rate": 7.432130030729804e-07,
- "loss": 0.05708106979727745,
- "num_tokens": 393029.0,
- "reward": 0.29566600918769836,
- "reward_std": 0.2818882167339325,
- "rewards/true_env_reward_fn/mean": 0.29566600918769836,
- "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
"step": 95,
- "step_time": 4.322851452001487
+ "step_time": 14.689755582000089
},
{
"clip_ratio/high_max": 0.0,
@@ -2676,26 +2676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0649794340133667,
- "epoch": 0.7804878048780488,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2012680470943451,
- "kl": 1.1399301456549438e-05,
- "learning_rate": 7.412536814109106e-07,
- "loss": -0.05478152632713318,
- "num_tokens": 398112.0,
- "reward": 0.23480799794197083,
- "reward_std": 0.28209570050239563,
- "rewards/true_env_reward_fn/mean": 0.23480799794197083,
- "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
"step": 96,
- "step_time": 3.4046103930013487
+ "step_time": 10.379233056999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2704,26 +2704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.5,
- "completions/mean_terminated_length": 56.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.4298859238624573,
- "epoch": 0.7886178861788617,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2239074409008026,
- "kl": 3.293174540885957e-05,
- "learning_rate": 7.392638036657332e-07,
- "loss": 0.09779056906700134,
- "num_tokens": 402892.0,
- "reward": 0.13796034455299377,
- "reward_std": 0.22141560912132263,
- "rewards/true_env_reward_fn/mean": 0.13796034455299377,
- "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
"step": 97,
- "step_time": 3.779275342998517
+ "step_time": 10.037491584999998
},
{
"clip_ratio/high_max": 0.0,
@@ -2732,26 +2732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 52.0,
- "completions/mean_terminated_length": 52.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2070425152778625,
- "epoch": 0.7967479674796748,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.19742031395435333,
- "kl": 1.4374184502230491e-05,
- "learning_rate": 7.372435480124337e-07,
- "loss": -0.006231316365301609,
- "num_tokens": 408052.0,
- "reward": 0.43320000171661377,
- "reward_std": 0.05237230286002159,
- "rewards/true_env_reward_fn/mean": 0.43320000171661377,
- "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
"step": 98,
- "step_time": 3.1304682769987267
+ "step_time": 13.073343929999965
},
{
"clip_ratio/high_max": 0.0,
@@ -2760,26 +2760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 0.9987849593162537,
- "epoch": 0.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.897383668227121e-05,
- "kl": 1.2614300885616103e-05,
- "learning_rate": 7.35193095346056e-07,
- "loss": 6.314263600870618e-07,
- "num_tokens": 409605.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
"step": 99,
- "step_time": 4.13536422299876
+ "step_time": 9.465850557000067
},
{
"clip_ratio/high_max": 0.0,
@@ -2788,26 +2788,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.0226224660873413,
- "epoch": 0.8130081300813008,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20178858935832977,
- "kl": 1.0500047665118473e-05,
- "learning_rate": 7.331126292655044e-07,
- "loss": -0.17970919609069824,
- "num_tokens": 411488.0,
- "reward": 0.6963247060775757,
- "reward_std": 0.18840119242668152,
- "rewards/true_env_reward_fn/mean": 0.6963247060775757,
- "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
"step": 100,
- "step_time": 3.7544156769981782
+ "step_time": 11.33050741000011
},
{
"clip_ratio/high_max": 0.0,
@@ -2816,26 +2816,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.0,
- "completions/mean_terminated_length": 59.0,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2509461045265198,
- "epoch": 0.8211382113821138,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22887632250785828,
- "kl": 2.1612477212329395e-05,
- "learning_rate": 7.310023360571047e-07,
- "loss": 0.025605827569961548,
- "num_tokens": 414080.0,
- "reward": 0.588032603263855,
- "reward_std": 0.11032751202583313,
- "rewards/true_env_reward_fn/mean": 0.588032603263855,
- "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
"step": 101,
- "step_time": 3.625197022998691
+ "step_time": 16.478299477000064
},
{
"clip_ratio/high_max": 0.0,
@@ -2844,26 +2844,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 176.0,
- "completions/max_terminated_length": 176.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5443179607391357,
- "epoch": 0.8292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10614532232284546,
- "kl": 1.492139062975184e-05,
- "learning_rate": 7.28862404677924e-07,
- "loss": 0.06531564146280289,
- "num_tokens": 419835.0,
- "reward": 0.07074306160211563,
- "reward_std": 0.2918013632297516,
- "rewards/true_env_reward_fn/mean": 0.07074306160211563,
- "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
"step": 102,
- "step_time": 7.796810614998321
+ "step_time": 11.344593008000174
},
{
"clip_ratio/high_max": 0.0,
@@ -2874,24 +2874,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 101.0,
"completions/max_terminated_length": 101.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3223788738250732,
- "epoch": 0.8373983739837398,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2421368807554245,
- "kl": 3.265505938543356e-05,
- "learning_rate": 7.266930267388503e-07,
- "loss": -0.07752113044261932,
- "num_tokens": 422773.0,
- "reward": 0.33568501472473145,
- "reward_std": 0.2780380845069885,
- "rewards/true_env_reward_fn/mean": 0.33568501472473145,
- "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
"step": 103,
- "step_time": 4.313938073000827
+ "step_time": 8.135681302999728
},
{
"clip_ratio/high_max": 0.0,
@@ -2900,26 +2900,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 67.25,
- "completions/mean_terminated_length": 67.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.3332037329673767,
- "epoch": 0.8455284552845529,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13883370161056519,
- "kl": 2.1224042484391248e-05,
- "learning_rate": 7.244943964874369e-07,
- "loss": 0.021739646792411804,
- "num_tokens": 426507.0,
- "reward": 0.40595096349716187,
- "reward_std": 0.2035457342863083,
- "rewards/true_env_reward_fn/mean": 0.40595096349716187,
- "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
"step": 104,
- "step_time": 4.155937195999286
+ "step_time": 15.185034105999875
},
{
"clip_ratio/high_max": 0.0,
@@ -2928,26 +2928,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
"completions/min_length": 31.0,
"completions/min_terminated_length": 31.0,
- "entropy": 1.3391229510307312,
- "epoch": 0.8536585365853658,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24205009639263153,
- "kl": 2.5022183763212524e-05,
- "learning_rate": 7.222667107905085e-07,
- "loss": 0.06330433487892151,
- "num_tokens": 429010.0,
- "reward": 0.3355163037776947,
- "reward_std": 0.2902730703353882,
- "rewards/true_env_reward_fn/mean": 0.3355163037776947,
- "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
"step": 105,
- "step_time": 3.808478789000219
+ "step_time": 9.588520330999927
},
{
"clip_ratio/high_max": 0.0,
@@ -2956,26 +2956,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1864720582962036,
- "epoch": 0.8617886178861789,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13473568856716156,
- "kl": 1.4212585938366828e-05,
- "learning_rate": 7.200101691165338e-07,
- "loss": -0.020715661346912384,
- "num_tokens": 432403.0,
- "reward": 0.4871198534965515,
- "reward_std": 0.15407639741897583,
- "rewards/true_env_reward_fn/mean": 0.4871198534965515,
- "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
"step": 106,
- "step_time": 4.240638332001254
+ "step_time": 14.60399662399982
},
{
"clip_ratio/high_max": 0.0,
@@ -2984,26 +2984,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.0,
- "completions/mean_terminated_length": 48.0,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "entropy": 1.0669284462928772,
- "epoch": 0.8699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14131899178028107,
- "kl": 1.5787159554747632e-05,
- "learning_rate": 7.177249735177651e-07,
- "loss": 0.03678784519433975,
- "num_tokens": 435995.0,
- "reward": 0.5010770559310913,
- "reward_std": 0.48966261744499207,
- "rewards/true_env_reward_fn/mean": 0.5010770559310913,
- "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
"step": 107,
- "step_time": 3.3587191269998584
+ "step_time": 20.100954443999854
},
{
"clip_ratio/high_max": 0.0,
@@ -3012,26 +3012,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 71.875,
- "completions/mean_terminated_length": 71.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.304731547832489,
- "epoch": 0.8780487804878049,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.380985673284158e-05,
- "kl": 1.3128728824085556e-05,
- "learning_rate": 7.154113286121462e-07,
- "loss": 6.494262834166875e-07,
- "num_tokens": 442094.0,
- "reward": 0.4055500030517578,
- "reward_std": 0.052258480340242386,
- "rewards/true_env_reward_fn/mean": 0.4055500030517578,
- "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
"step": 108,
- "step_time": 4.337008413998774
+ "step_time": 8.304149297999857
},
{
"clip_ratio/high_max": 0.0,
@@ -3040,26 +3040,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2324069738388062,
- "epoch": 0.8861788617886179,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13588950037956238,
- "kl": 1.3448377558233915e-05,
- "learning_rate": 7.130694415649912e-07,
- "loss": 1.0952353477478027e-06,
- "num_tokens": 447226.0,
- "reward": 0.20854972302913666,
- "reward_std": 0.06059705466032028,
- "rewards/true_env_reward_fn/mean": 0.20854972302913666,
- "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
"step": 109,
- "step_time": 3.2976038649994734
+ "step_time": 11.260021517000041
},
{
"clip_ratio/high_max": 0.0,
@@ -3068,26 +3068,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1567262411117554,
- "epoch": 0.8943089430894309,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.736967720324174e-05,
- "kl": 1.2838129805459175e-05,
- "learning_rate": 7.106995220704342e-07,
- "loss": 6.425898391171359e-07,
- "num_tokens": 450359.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
"step": 110,
- "step_time": 4.067084037998939
+ "step_time": 22.80178854000019
},
{
"clip_ratio/high_max": 0.0,
@@ -3098,24 +3098,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 100.0,
"completions/max_terminated_length": 100.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 12.0,
- "completions/min_terminated_length": 12.0,
- "entropy": 1.496058464050293,
- "epoch": 0.9024390243902439,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18041981756687164,
- "kl": 1.6616825632809196e-05,
- "learning_rate": 7.083017823326532e-07,
- "loss": 0.0269068144261837,
- "num_tokens": 453583.0,
- "reward": 0.5647265911102295,
- "reward_std": 0.1507105529308319,
- "rewards/true_env_reward_fn/mean": 0.5647265911102295,
- "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
"step": 111,
- "step_time": 4.347732382997492
+ "step_time": 9.663163859000178
},
{
"clip_ratio/high_max": 0.0,
@@ -3124,26 +3124,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 67.75,
- "completions/mean_terminated_length": 67.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.4196155667304993,
- "epoch": 0.9105691056910569,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18451112508773804,
- "kl": 2.1803500203532167e-05,
- "learning_rate": 7.058764370468698e-07,
- "loss": 0.1650262475013733,
- "num_tokens": 456773.0,
- "reward": 0.6907394528388977,
- "reward_std": 0.1393815129995346,
- "rewards/true_env_reward_fn/mean": 0.6907394528388977,
- "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
"step": 112,
- "step_time": 4.627644968999448
+ "step_time": 9.903081222999617
},
{
"clip_ratio/high_max": 0.0,
@@ -3152,26 +3152,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2243221998214722,
- "epoch": 0.9186991869918699,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331371545791626,
- "kl": 1.548633599668392e-05,
- "learning_rate": 7.034237033801247e-07,
- "loss": 0.039844345301389694,
- "num_tokens": 462714.0,
- "reward": 0.21676866710186005,
- "reward_std": 0.26559779047966003,
- "rewards/true_env_reward_fn/mean": 0.21676866710186005,
- "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
"step": 113,
- "step_time": 3.8455466220002563
+ "step_time": 10.101770388999284
},
{
"clip_ratio/high_max": 0.0,
@@ -3180,26 +3180,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 73.0,
- "completions/mean_terminated_length": 73.0,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.325823724269867,
- "epoch": 0.926829268292683,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1866220384836197,
- "kl": 1.8801019905367866e-05,
- "learning_rate": 7.009438009518325e-07,
- "loss": 0.06504581868648529,
- "num_tokens": 465994.0,
- "reward": 0.5194582939147949,
- "reward_std": 0.2796703577041626,
- "rewards/true_env_reward_fn/mean": 0.5194582939147949,
- "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
"step": 114,
- "step_time": 4.151028698999653
+ "step_time": 17.633509853000305
},
{
"clip_ratio/high_max": 0.0,
@@ -3208,26 +3208,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.1521879434585571,
- "epoch": 0.9349593495934959,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14320029318332672,
- "kl": 1.2749982033710694e-05,
- "learning_rate": 6.98436951814117e-07,
- "loss": 0.03685300797224045,
- "num_tokens": 468615.0,
- "reward": 0.5399107933044434,
- "reward_std": 0.26432597637176514,
- "rewards/true_env_reward_fn/mean": 0.5399107933044434,
- "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
"step": 115,
- "step_time": 3.6973990600017714
+ "step_time": 12.307247350000125
},
{
"clip_ratio/high_max": 0.0,
@@ -3236,26 +3236,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.201507806777954,
- "epoch": 0.943089430894309,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.14264807105064392,
- "kl": 2.6679515940486453e-05,
- "learning_rate": 6.959033804319283e-07,
- "loss": -0.023484818637371063,
- "num_tokens": 471647.0,
- "reward": 0.41836902499198914,
- "reward_std": 0.3116860091686249,
- "rewards/true_env_reward_fn/mean": 0.41836902499198914,
- "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
"step": 116,
- "step_time": 3.1295652919998247
+ "step_time": 14.468690254000194
},
{
"clip_ratio/high_max": 0.0,
@@ -3264,26 +3264,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 62.375,
- "completions/mean_terminated_length": 62.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2834057807922363,
- "epoch": 0.9512195121951219,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.810227154754102e-05,
- "kl": 1.4841665233689127e-05,
- "learning_rate": 6.933433136629443e-07,
- "loss": 7.425555850204546e-07,
- "num_tokens": 474682.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
"step": 117,
- "step_time": 3.4368692790012574
+ "step_time": 11.849063975999798
},
{
"clip_ratio/high_max": 0.0,
@@ -3292,26 +3292,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.191932499408722,
- "epoch": 0.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21204856038093567,
- "kl": 3.64198385796044e-05,
- "learning_rate": 6.907569807372574e-07,
- "loss": -0.001312553882598877,
- "num_tokens": 477027.0,
- "reward": 0.5300568342208862,
- "reward_std": 0.2945883274078369,
- "rewards/true_env_reward_fn/mean": 0.5300568342208862,
- "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
"step": 118,
- "step_time": 3.8569856240010267
+ "step_time": 16.597334930000216
},
{
"clip_ratio/high_max": 0.0,
@@ -3320,26 +3320,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 190.0,
- "completions/max_terminated_length": 190.0,
- "completions/mean_length": 96.5,
- "completions/mean_terminated_length": 96.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2401175498962402,
- "epoch": 0.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011377666669432074,
- "kl": 1.3742283954343293e-05,
- "learning_rate": 6.881446132368494e-07,
- "loss": 6.866695230201003e-07,
- "num_tokens": 481999.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
"step": 119,
- "step_time": 8.09440958399864
+ "step_time": 11.729475523999554
},
{
"clip_ratio/high_max": 0.0,
@@ -3348,26 +3348,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.230682611465454,
- "epoch": 0.975609756097561,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22175048291683197,
- "kl": 1.2522132237791084e-05,
- "learning_rate": 6.855064450748555e-07,
- "loss": -0.04083740711212158,
- "num_tokens": 490884.0,
- "reward": 0.13476666808128357,
- "reward_std": 0.2987530529499054,
- "rewards/true_env_reward_fn/mean": 0.13476666808128357,
- "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
"step": 120,
- "step_time": 4.678523641001448
+ "step_time": 11.503627788000358
},
{
"clip_ratio/high_max": 0.0,
@@ -3376,26 +3376,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2799639105796814,
- "epoch": 0.983739837398374,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19926966726779938,
- "kl": 1.7022688552970067e-05,
- "learning_rate": 6.828427124746189e-07,
- "loss": -0.010804429650306702,
- "num_tokens": 496404.0,
- "reward": 0.24633333086967468,
- "reward_std": 0.2454334795475006,
- "rewards/true_env_reward_fn/mean": 0.24633333086967468,
- "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
"step": 121,
- "step_time": 3.98071062300005
+ "step_time": 8.728293746000418
},
{
"clip_ratio/high_max": 0.0,
@@ -3404,26 +3404,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 46.875,
- "completions/mean_terminated_length": 46.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3840235471725464,
- "epoch": 0.991869918699187,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24853822588920593,
- "kl": 3.688259130285587e-05,
- "learning_rate": 6.801536539485403e-07,
- "loss": 0.10205884277820587,
- "num_tokens": 499767.0,
- "reward": 0.3045905530452728,
- "reward_std": 0.262839138507843,
- "rewards/true_env_reward_fn/mean": 0.3045905530452728,
- "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
"step": 122,
- "step_time": 3.3792565210005705
+ "step_time": 15.51957702100026
},
{
"clip_ratio/high_max": 0.0,
@@ -3432,26 +3432,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 50.375,
- "completions/mean_terminated_length": 50.375,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2064164280891418,
- "epoch": 1.0,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1365528553724289,
- "kl": 2.434901080050622e-05,
- "learning_rate": 6.774395102767203e-07,
- "loss": -0.03472680225968361,
- "num_tokens": 504906.0,
- "reward": 0.2722649872303009,
- "reward_std": 0.2922348082065582,
- "rewards/true_env_reward_fn/mean": 0.2722649872303009,
- "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
"step": 123,
- "step_time": 3.0233660449994204
+ "step_time": 8.471463828000196
},
{
"clip_ratio/high_max": 0.0,
@@ -3460,26 +3460,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2819936871528625,
- "epoch": 1.008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00023045104171615094,
- "kl": 2.2608143808611203e-05,
- "learning_rate": 6.747005244854004e-07,
- "loss": 1.1284330412308918e-06,
- "num_tokens": 508329.0,
- "reward": 0.3149532079696655,
- "reward_std": 0.4275679290294647,
- "rewards/true_env_reward_fn/mean": 0.3149532079696655,
- "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
"step": 124,
- "step_time": 4.01701365199915
+ "step_time": 13.438758649999727
},
{
"clip_ratio/high_max": 0.0,
@@ -3488,26 +3488,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 157.0,
- "completions/max_terminated_length": 157.0,
- "completions/mean_length": 80.875,
- "completions/mean_terminated_length": 80.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1542360186576843,
- "epoch": 1.016260162601626,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.232049003709108e-05,
- "kl": 1.291002809011843e-05,
- "learning_rate": 6.719369418252023e-07,
- "loss": 6.488799613180163e-07,
- "num_tokens": 515076.0,
- "reward": 0.4841846525669098,
- "reward_std": 0.12780573964118958,
- "rewards/true_env_reward_fn/mean": 0.4841846525669098,
- "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
"step": 125,
- "step_time": 7.240956699999515
+ "step_time": 8.654177170999901
},
{
"clip_ratio/high_max": 0.0,
@@ -3516,26 +3516,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2594389915466309,
- "epoch": 1.024390243902439,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13091468811035156,
- "kl": 1.751603304001037e-05,
- "learning_rate": 6.691490097491675e-07,
- "loss": -0.033413223922252655,
- "num_tokens": 520279.0,
- "reward": 0.28095200657844543,
- "reward_std": 0.21837711334228516,
- "rewards/true_env_reward_fn/mean": 0.28095200657844543,
- "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
"step": 126,
- "step_time": 3.355879656997786
+ "step_time": 10.513378469000145
},
{
"clip_ratio/high_max": 0.0,
@@ -3544,26 +3544,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 136.0,
- "completions/max_terminated_length": 136.0,
- "completions/mean_length": 77.875,
- "completions/mean_terminated_length": 77.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.3044686317443848,
- "epoch": 1.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12953205406665802,
- "kl": 1.8700401597016025e-05,
- "learning_rate": 6.663369778906008e-07,
- "loss": 0.03562816232442856,
- "num_tokens": 524582.0,
- "reward": 0.4330660402774811,
- "reward_std": 0.4592672288417816,
- "rewards/true_env_reward_fn/mean": 0.4330660402774811,
- "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
"step": 127,
- "step_time": 5.965807722999671
+ "step_time": 23.20779430600078
},
{
"clip_ratio/high_max": 0.0,
@@ -3572,26 +3572,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 117.0,
- "completions/max_terminated_length": 117.0,
- "completions/mean_length": 75.375,
- "completions/mean_terminated_length": 75.375,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1742327809333801,
- "epoch": 1.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14993594586849213,
- "kl": 1.1459212601039326e-05,
- "learning_rate": 6.635010980407174e-07,
- "loss": 0.03646668791770935,
- "num_tokens": 526213.0,
- "reward": 0.7185037136077881,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.7185037136077881,
- "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
"step": 128,
- "step_time": 4.9305356690001645
+ "step_time": 11.551069149999876
},
{
"clip_ratio/high_max": 0.0,
@@ -3600,26 +3600,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 49.125,
- "completions/mean_terminated_length": 49.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0784690976142883,
- "epoch": 1.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16007214784622192,
- "kl": 1.2491957932070363e-05,
- "learning_rate": 6.606416241260979e-07,
- "loss": 0.006608985364437103,
- "num_tokens": 531862.0,
- "reward": 0.2934249937534332,
- "reward_std": 0.2395382523536682,
- "rewards/true_env_reward_fn/mean": 0.2934249937534332,
- "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
"step": 129,
- "step_time": 3.173622508000335
+ "step_time": 10.723005456000465
},
{
"clip_ratio/high_max": 0.0,
@@ -3628,26 +3628,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.308219850063324,
- "epoch": 1.056910569105691,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.985446427483112e-05,
- "kl": 1.2420873190421844e-05,
- "learning_rate": 6.577588121859508e-07,
- "loss": 6.241918413252279e-07,
- "num_tokens": 535957.0,
- "reward": 0.4817493259906769,
- "reward_std": 0.029202036559581757,
- "rewards/true_env_reward_fn/mean": 0.4817493259906769,
- "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
"step": 130,
- "step_time": 4.251137947001553
+ "step_time": 15.15810051499966
},
{
"clip_ratio/high_max": 0.0,
@@ -3656,26 +3656,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.0767641067504883,
- "epoch": 1.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15786457061767578,
- "kl": 1.8847958926926367e-05,
- "learning_rate": 6.548529203491875e-07,
- "loss": -0.0026272237300872803,
- "num_tokens": 539269.0,
- "reward": 0.536803662776947,
- "reward_std": 0.30375123023986816,
- "rewards/true_env_reward_fn/mean": 0.536803662776947,
- "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
"step": 131,
- "step_time": 3.7980547870010923
+ "step_time": 20.109428818001106
},
{
"clip_ratio/high_max": 0.0,
@@ -3684,26 +3684,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3295028805732727,
- "epoch": 1.0731707317073171,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1881481558084488,
- "kl": 2.0969039724150207e-05,
- "learning_rate": 6.519242088113085e-07,
- "loss": 0.08431969583034515,
- "num_tokens": 545691.0,
- "reward": 0.24590599536895752,
- "reward_std": 0.2047487199306488,
- "rewards/true_env_reward_fn/mean": 0.24590599536895752,
- "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
"step": 132,
- "step_time": 4.361092664001262
+ "step_time": 10.12403799699996
},
{
"clip_ratio/high_max": 0.0,
@@ -3712,26 +3712,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2131375670433044,
- "epoch": 1.08130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13930389285087585,
- "kl": 1.1046585314034019e-05,
- "learning_rate": 6.489729398111058e-07,
- "loss": -0.03801802545785904,
- "num_tokens": 550295.0,
- "reward": 0.3215479254722595,
- "reward_std": 0.1736886352300644,
- "rewards/true_env_reward_fn/mean": 0.3215479254722595,
- "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
"step": 133,
- "step_time": 3.372364626999115
+ "step_time": 10.396350653999434
},
{
"clip_ratio/high_max": 0.0,
@@ -3740,26 +3740,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 63.375,
- "completions/mean_terminated_length": 63.375,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2786019444465637,
- "epoch": 1.089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12932609021663666,
- "kl": 1.340499647994875e-05,
- "learning_rate": 6.459993776071815e-07,
- "loss": 0.029022663831710815,
- "num_tokens": 553826.0,
- "reward": 0.4830188751220703,
- "reward_std": 0.29014864563941956,
- "rewards/true_env_reward_fn/mean": 0.4830188751220703,
- "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
"step": 134,
- "step_time": 3.215292060998763
+ "step_time": 10.961974539000039
},
{
"clip_ratio/high_max": 0.0,
@@ -3768,26 +3768,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 49.375,
- "completions/mean_terminated_length": 49.375,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 0.9003906548023224,
- "epoch": 1.0975609756097562,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15958240628242493,
- "kl": 2.838099044311093e-05,
- "learning_rate": 6.430037884542861e-07,
- "loss": 0.11459673941135406,
- "num_tokens": 557217.0,
- "reward": 0.494448184967041,
- "reward_std": 0.3076546788215637,
- "rewards/true_env_reward_fn/mean": 0.494448184967041,
- "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
"step": 135,
- "step_time": 3.500462582000182
+ "step_time": 11.779171687000144
},
{
"clip_ratio/high_max": 0.0,
@@ -3798,24 +3798,24 @@
"completions/clipped_ratio": 0.0,
"completions/max_length": 93.0,
"completions/max_terminated_length": 93.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2928712964057922,
- "epoch": 1.1056910569105691,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23077522218227386,
- "kl": 2.251418845844455e-05,
- "learning_rate": 6.399864405794782e-07,
- "loss": -0.05874824523925781,
- "num_tokens": 562421.0,
- "reward": 0.2385583370923996,
- "reward_std": 0.23380905389785767,
- "rewards/true_env_reward_fn/mean": 0.2385583370923996,
- "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
"step": 136,
- "step_time": 4.208805245998519
+ "step_time": 8.779588141000204
},
{
"clip_ratio/high_max": 0.0,
@@ -3824,26 +3824,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8736326098442078,
- "epoch": 1.113821138211382,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.871674071997404e-05,
- "kl": 1.1485328741400735e-05,
- "learning_rate": 6.369476041581066e-07,
- "loss": 5.747077125306532e-07,
- "num_tokens": 566387.0,
- "reward": 0.4902166724205017,
- "reward_std": 0.038254011422395706,
- "rewards/true_env_reward_fn/mean": 0.4902166724205017,
- "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
"step": 137,
- "step_time": 3.981489739000608
+ "step_time": 11.74765996799988
},
{
"clip_ratio/high_max": 0.0,
@@ -3852,26 +3852,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1538971662521362,
- "epoch": 1.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14676779508590698,
- "kl": 1.1651037766569061e-05,
- "learning_rate": 6.338875512896188e-07,
- "loss": 0.1347643882036209,
- "num_tokens": 569341.0,
- "reward": 0.43844783306121826,
- "reward_std": 0.16067014634609222,
- "rewards/true_env_reward_fn/mean": 0.43844783306121826,
- "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
"step": 138,
- "step_time": 3.689221037999232
+ "step_time": 8.8135579650002
},
{
"clip_ratio/high_max": 0.0,
@@ -3880,26 +3880,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.625,
- "completions/mean_terminated_length": 54.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1961707472801208,
- "epoch": 1.1300813008130082,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16650564968585968,
- "kl": 1.4349476259667426e-05,
- "learning_rate": 6.308065559731976e-07,
- "loss": 0.007910434156656265,
- "num_tokens": 574046.0,
- "reward": 0.4596000015735626,
- "reward_std": 0.07715634256601334,
- "rewards/true_env_reward_fn/mean": 0.4596000015735626,
- "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
"step": 139,
- "step_time": 3.6711935700004688
+ "step_time": 9.997661417000472
},
{
"clip_ratio/high_max": 0.0,
@@ -3908,26 +3908,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2013322114944458,
- "epoch": 1.1382113821138211,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14415834844112396,
- "kl": 1.4664098671346437e-05,
- "learning_rate": 6.277048940832264e-07,
- "loss": -0.016162052750587463,
- "num_tokens": 576769.0,
- "reward": 0.6152583360671997,
- "reward_std": 0.07727260142564774,
- "rewards/true_env_reward_fn/mean": 0.6152583360671997,
- "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
"step": 140,
- "step_time": 3.5191362610003125
+ "step_time": 14.679971276999822
},
{
"clip_ratio/high_max": 0.0,
@@ -3936,26 +3936,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 44.75,
- "completions/mean_terminated_length": 44.75,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0287770330905914,
- "epoch": 1.146341463414634,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.892051457427442e-05,
- "kl": 1.1797974821092794e-05,
- "learning_rate": 6.245828433445872e-07,
- "loss": 5.92092192164273e-07,
- "num_tokens": 578843.0,
- "reward": 0.6387845277786255,
- "reward_std": 0.13867565989494324,
- "rewards/true_env_reward_fn/mean": 0.6387845277786255,
- "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
"step": 141,
- "step_time": 2.5015027329991426
+ "step_time": 11.64747691499997
},
{
"clip_ratio/high_max": 0.0,
@@ -3964,26 +3964,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1590029001235962,
- "epoch": 1.1544715447154472,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.160966694355011,
- "kl": 1.4735675904375967e-05,
- "learning_rate": 6.214406833077937e-07,
- "loss": 0.0170527845621109,
- "num_tokens": 583201.0,
- "reward": 0.36017733812332153,
- "reward_std": 0.3556094467639923,
- "rewards/true_env_reward_fn/mean": 0.36017733812332153,
- "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
"step": 142,
- "step_time": 3.2783409929998015
+ "step_time": 8.380270293999729
},
{
"clip_ratio/high_max": 0.0,
@@ -3992,26 +3992,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 59.0,
- "completions/min_terminated_length": 59.0,
- "entropy": 1.1985241174697876,
- "epoch": 1.1626016260162602,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12272457778453827,
- "kl": 1.7849098185251933e-05,
- "learning_rate": 6.182786953239593e-07,
- "loss": -0.0016125142574310303,
- "num_tokens": 587317.0,
- "reward": 0.34745320677757263,
- "reward_std": 0.3954337239265442,
- "rewards/true_env_reward_fn/mean": 0.34745320677757263,
- "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
"step": 143,
- "step_time": 3.9932043310000154
+ "step_time": 11.716556537000088
},
{
"clip_ratio/high_max": 0.0,
@@ -4020,26 +4020,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 48.0,
- "completions/max_terminated_length": 48.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1116944551467896,
- "epoch": 1.170731707317073,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18435250222682953,
- "kl": 1.014559029499651e-05,
- "learning_rate": 6.150971625196048e-07,
- "loss": 0.009793907403945923,
- "num_tokens": 590191.0,
- "reward": 0.4938516616821289,
- "reward_std": 0.03703190013766289,
- "rewards/true_env_reward_fn/mean": 0.4938516616821289,
- "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
"step": 144,
- "step_time": 2.3663663690022076
+ "step_time": 10.67718802499985
},
{
"clip_ratio/high_max": 0.0,
@@ -4048,26 +4048,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 53.0,
- "completions/mean_terminated_length": 53.0,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.413200855255127,
- "epoch": 1.1788617886178863,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14336225390434265,
- "kl": 2.0541991034406237e-05,
- "learning_rate": 6.118963697713078e-07,
- "loss": -0.013927727937698364,
- "num_tokens": 593671.0,
- "reward": 0.4619143605232239,
- "reward_std": 0.3773181140422821,
- "rewards/true_env_reward_fn/mean": 0.4619143605232239,
- "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
"step": 145,
- "step_time": 3.9730388410007436
+ "step_time": 8.610043666999445
},
{
"clip_ratio/high_max": 0.0,
@@ -4076,26 +4076,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 128.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 6.0,
- "completions/min_terminated_length": 6.0,
- "entropy": 1.08676016330719,
- "epoch": 1.1869918699186992,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15555191040039062,
- "kl": 1.6947700260061538e-05,
- "learning_rate": 6.086766036801937e-07,
- "loss": -0.139797180891037,
- "num_tokens": 601612.0,
- "reward": 0.3831036686897278,
- "reward_std": 0.09242849797010422,
- "rewards/true_env_reward_fn/mean": 0.3831036686897278,
- "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
"step": 146,
- "step_time": 6.323679949000507
+ "step_time": 13.649344002000362
},
{
"clip_ratio/high_max": 0.0,
@@ -4104,26 +4104,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.5055813789367676,
- "epoch": 1.1951219512195121,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2397669553756714,
- "kl": 2.704876442294335e-05,
- "learning_rate": 6.054381525462745e-07,
- "loss": 0.2738838493824005,
- "num_tokens": 606712.0,
- "reward": 0.25339600443840027,
- "reward_std": 0.3023079037666321,
- "rewards/true_env_reward_fn/mean": 0.25339600443840027,
- "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
"step": 147,
- "step_time": 5.185072233998653
+ "step_time": 11.702765863000877
},
{
"clip_ratio/high_max": 0.0,
@@ -4132,26 +4132,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 47.25,
- "completions/mean_terminated_length": 47.25,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.135968267917633,
- "epoch": 1.203252032520325,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2614514231681824,
- "kl": 3.613240005506668e-05,
- "learning_rate": 6.021813063426323e-07,
- "loss": 0.10286401212215424,
- "num_tokens": 610566.0,
- "reward": 0.31031692028045654,
- "reward_std": 0.3124054968357086,
- "rewards/true_env_reward_fn/mean": 0.31031692028045654,
- "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
"step": 148,
- "step_time": 3.2177847610000754
+ "step_time": 13.306644664000487
},
{
"clip_ratio/high_max": 0.0,
@@ -4160,26 +4160,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 52.75,
- "completions/mean_terminated_length": 52.75,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4589928984642029,
- "epoch": 1.2113821138211383,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2019941508769989,
- "kl": 2.1841721718374174e-05,
- "learning_rate": 5.989063566894572e-07,
- "loss": 0.010915875434875488,
- "num_tokens": 615716.0,
- "reward": 0.31711751222610474,
- "reward_std": 0.13289952278137207,
- "rewards/true_env_reward_fn/mean": 0.31711751222610474,
- "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
"step": 149,
- "step_time": 4.3804878079990885
+ "step_time": 9.446422488999815
},
{
"clip_ratio/high_max": 0.0,
@@ -4188,26 +4188,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.1892729997634888,
- "epoch": 1.2195121951219512,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.835455471649766e-05,
- "kl": 1.3420096820482286e-05,
- "learning_rate": 5.956135968279332e-07,
- "loss": 6.646802717114042e-07,
- "num_tokens": 619439.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
"step": 150,
- "step_time": 3.63938895299907
+ "step_time": 10.538116119999813
},
{
"clip_ratio/high_max": 0.0,
@@ -4216,26 +4216,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1402934789657593,
- "epoch": 1.2276422764227641,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010795716661959887,
- "kl": 1.4652535810455447e-05,
- "learning_rate": 5.923033215939834e-07,
- "loss": 7.542968205598299e-07,
- "num_tokens": 621009.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
"step": 151,
- "step_time": 3.926544339999964
+ "step_time": 15.755764130999978
},
{
"clip_ratio/high_max": 0.0,
@@ -4244,26 +4244,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
"completions/mean_length": 64.125,
"completions/mean_terminated_length": 64.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.190350890159607,
- "epoch": 1.2357723577235773,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19853363931179047,
- "kl": 2.269768037876929e-05,
- "learning_rate": 5.889758273918683e-07,
- "loss": 0.044217392802238464,
- "num_tokens": 623994.0,
- "reward": 0.4411996603012085,
- "reward_std": 0.2517909109592438,
- "rewards/true_env_reward_fn/mean": 0.4411996603012085,
- "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
"step": 152,
- "step_time": 3.7339736520007136
+ "step_time": 9.670861957999932
},
{
"clip_ratio/high_max": 0.0,
@@ -4272,26 +4272,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.2316884994506836,
- "epoch": 1.2439024390243902,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.35867181699723e-05,
- "kl": 1.1579370038816705e-05,
- "learning_rate": 5.856314121676467e-07,
- "loss": 5.79387460675207e-07,
- "num_tokens": 628224.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
"step": 153,
- "step_time": 4.375236807001784
+ "step_time": 22.82054339000024
},
{
"clip_ratio/high_max": 0.0,
@@ -4300,26 +4300,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 137.0,
- "completions/max_terminated_length": 137.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.4028943181037903,
- "epoch": 1.2520325203252032,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25418931245803833,
- "kl": 3.6890452065563295e-05,
- "learning_rate": 5.822703753824966e-07,
- "loss": 0.25599968433380127,
- "num_tokens": 631183.0,
- "reward": 0.38683533668518066,
- "reward_std": 0.43613559007644653,
- "rewards/true_env_reward_fn/mean": 0.38683533668518066,
- "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
"step": 154,
- "step_time": 6.055355972999678
+ "step_time": 9.973958625999785
},
{
"clip_ratio/high_max": 0.0,
@@ -4328,26 +4328,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.367663562297821,
- "epoch": 1.2601626016260163,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13331811130046844,
- "kl": 1.2863993106293492e-05,
- "learning_rate": 5.788930179859024e-07,
- "loss": -0.0047044456005096436,
- "num_tokens": 636230.0,
- "reward": 0.3489508628845215,
- "reward_std": 0.13627417385578156,
- "rewards/true_env_reward_fn/mean": 0.3489508628845215,
- "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
"step": 155,
- "step_time": 3.9174396130001696
+ "step_time": 9.758407419000378
},
{
"clip_ratio/high_max": 0.0,
@@ -4356,26 +4356,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 135.0,
- "completions/max_terminated_length": 135.0,
- "completions/mean_length": 68.5,
- "completions/mean_terminated_length": 68.5,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.2759611010551453,
- "epoch": 1.2682926829268293,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12738144397735596,
- "kl": 1.5844128029129934e-05,
- "learning_rate": 5.754996423887061e-07,
- "loss": -0.011055335402488708,
- "num_tokens": 640262.0,
- "reward": 0.344404935836792,
- "reward_std": 0.16542991995811462,
- "rewards/true_env_reward_fn/mean": 0.344404935836792,
- "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
"step": 156,
- "step_time": 5.706334413998775
+ "step_time": 11.009583763999672
},
{
"clip_ratio/high_max": 0.0,
@@ -4384,26 +4384,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.0904476642608643,
- "epoch": 1.2764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.733699021628127e-05,
- "kl": 1.2238857834745431e-05,
- "learning_rate": 5.720905524360308e-07,
- "loss": 6.076299996493617e-07,
- "num_tokens": 645091.0,
- "reward": 0.4731999933719635,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4731999933719635,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
"step": 157,
- "step_time": 3.7696847109982627
+ "step_time": 10.12789283499933
},
{
"clip_ratio/high_max": 0.0,
@@ -4412,26 +4412,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 60.0,
- "completions/mean_terminated_length": 60.0,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.3856677412986755,
- "epoch": 1.2845528455284554,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18670028448104858,
- "kl": 1.8415606064081658e-05,
- "learning_rate": 5.686660533800736e-07,
- "loss": -0.07078710198402405,
- "num_tokens": 648179.0,
- "reward": 0.537517786026001,
- "reward_std": 0.1451217085123062,
- "rewards/true_env_reward_fn/mean": 0.537517786026001,
- "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
"step": 158,
- "step_time": 3.7075291149994882
+ "step_time": 7.800485663000472
},
{
"clip_ratio/high_max": 0.0,
@@ -4440,26 +4440,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 68.125,
- "completions/mean_terminated_length": 68.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.121916651725769,
- "epoch": 1.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11415883898735046,
- "kl": 1.909901220642496e-05,
- "learning_rate": 5.652264518527725e-07,
- "loss": -0.04401372745633125,
- "num_tokens": 652044.0,
- "reward": 0.5182899832725525,
- "reward_std": 0.21869486570358276,
- "rewards/true_env_reward_fn/mean": 0.5182899832725525,
- "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
"step": 159,
- "step_time": 3.8929355969994504
+ "step_time": 13.77649076500029
},
{
"clip_ratio/high_max": 0.0,
@@ -4468,26 +4468,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 143.0,
- "completions/max_terminated_length": 143.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5205118060112,
- "epoch": 1.3008130081300813,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2403375506401062,
- "kl": 3.7574073758150917e-05,
- "learning_rate": 5.617720558383508e-07,
- "loss": 0.26385918259620667,
- "num_tokens": 656362.0,
- "reward": 0.41201668977737427,
- "reward_std": 0.2023741453886032,
- "rewards/true_env_reward_fn/mean": 0.41201668977737427,
- "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
"step": 160,
- "step_time": 6.157555950998358
+ "step_time": 13.25029361300085
},
{
"clip_ratio/high_max": 0.0,
@@ -4496,26 +4496,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3556928038597107,
- "epoch": 1.3089430894308944,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18135924637317657,
- "kl": 2.6372636057203636e-05,
- "learning_rate": 5.583031746457407e-07,
- "loss": -0.10538280755281448,
- "num_tokens": 659977.0,
- "reward": 0.4239906072616577,
- "reward_std": 0.3287450969219208,
- "rewards/true_env_reward_fn/mean": 0.4239906072616577,
- "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
"step": 161,
- "step_time": 4.709477423999488
+ "step_time": 17.837881629000094
},
{
"clip_ratio/high_max": 0.0,
@@ -4524,26 +4524,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 52.5,
- "completions/mean_terminated_length": 52.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 0.9743769466876984,
- "epoch": 1.3170731707317074,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.178797647356987,
- "kl": 1.2532927030406427e-05,
- "learning_rate": 5.548201188808869e-07,
- "loss": -0.04164513945579529,
- "num_tokens": 661409.0,
- "reward": 0.8041956424713135,
- "reward_std": 0.1363772451877594,
- "rewards/true_env_reward_fn/mean": 0.8041956424713135,
- "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
"step": 162,
- "step_time": 3.1791253910014348
+ "step_time": 9.286757633000889
},
{
"clip_ratio/high_max": 0.0,
@@ -4552,26 +4552,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.3072250485420227,
- "epoch": 1.3252032520325203,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16768279671669006,
- "kl": 1.89386219062726e-05,
- "learning_rate": 5.513232004189339e-07,
- "loss": -0.01292814314365387,
- "num_tokens": 666504.0,
- "reward": 0.27981066703796387,
- "reward_std": 0.2949208915233612,
- "rewards/true_env_reward_fn/mean": 0.27981066703796387,
- "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
"step": 163,
- "step_time": 3.778431355000066
+ "step_time": 10.785562561999996
},
{
"clip_ratio/high_max": 0.0,
@@ -4580,26 +4580,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 51.0,
- "completions/mean_terminated_length": 51.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1984856128692627,
- "epoch": 1.3333333333333333,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13123764097690582,
- "kl": 1.9091786271019373e-05,
- "learning_rate": 5.478127323763027e-07,
- "loss": 0.035523779690265656,
- "num_tokens": 671004.0,
- "reward": 0.2771500051021576,
- "reward_std": 0.30146247148513794,
- "rewards/true_env_reward_fn/mean": 0.2771500051021576,
- "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
"step": 164,
- "step_time": 3.304021460000513
+ "step_time": 12.722446307000155
},
{
"clip_ratio/high_max": 0.0,
@@ -4608,26 +4608,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 102.0,
- "completions/max_terminated_length": 102.0,
- "completions/mean_length": 84.875,
- "completions/mean_terminated_length": 84.875,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2726752758026123,
- "epoch": 1.3414634146341464,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15594874322414398,
- "kl": 2.5981638827943243e-05,
- "learning_rate": 5.442890290826518e-07,
- "loss": -0.01398652046918869,
- "num_tokens": 677307.0,
- "reward": 0.35573017597198486,
- "reward_std": 0.25944042205810547,
- "rewards/true_env_reward_fn/mean": 0.35573017597198486,
- "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
"step": 165,
- "step_time": 4.972808451999299
+ "step_time": 14.89655208000022
},
{
"clip_ratio/high_max": 0.0,
@@ -4636,26 +4636,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 210.0,
- "completions/max_terminated_length": 210.0,
- "completions/mean_length": 103.625,
- "completions/mean_terminated_length": 103.625,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.0949090719223022,
- "epoch": 1.3495934959349594,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.07191785424947739,
- "kl": 1.4828182884230046e-05,
- "learning_rate": 5.407524060527332e-07,
- "loss": -0.05063021928071976,
- "num_tokens": 682076.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
"step": 166,
- "step_time": 8.769379133000257
+ "step_time": 8.962532588000613
},
{
"clip_ratio/high_max": 0.0,
@@ -4664,54 +4664,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1857684254646301,
- "epoch": 1.3577235772357723,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.923542191041633e-05,
- "kl": 1.0165251751459436e-05,
- "learning_rate": 5.37203179958141e-07,
- "loss": 5.114516170579009e-07,
- "num_tokens": 685500.0,
- "reward": 0.5610077381134033,
- "reward_std": 0.316459059715271,
- "rewards/true_env_reward_fn/mean": 0.5610077381134033,
- "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
"step": 167,
- "step_time": 3.1906087530005607
+ "step_time": 11.708963983000103
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 74.25,
- "completions/mean_terminated_length": 74.25,
- "completions/min_length": 16.0,
- "completions/min_terminated_length": 16.0,
- "entropy": 1.527149498462677,
- "epoch": 1.3658536585365852,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13164250552654266,
- "kl": 1.8541333702160046e-05,
- "learning_rate": 5.33641668598956e-07,
- "loss": -0.2347300797700882,
- "num_tokens": 688318.0,
- "reward": 0.7218117713928223,
- "reward_std": 0.1818692982196808,
- "rewards/true_env_reward_fn/mean": 0.7218117713928223,
- "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
"step": 168,
- "step_time": 8.50137474999974
+ "step_time": 16.905241970000134
},
{
"clip_ratio/high_max": 0.0,
@@ -4720,26 +4720,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 63.0,
- "completions/min_terminated_length": 63.0,
- "entropy": 1.3749513030052185,
- "epoch": 1.3739837398373984,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13246971368789673,
- "kl": 1.2620409506780561e-05,
- "learning_rate": 5.300681908752895e-07,
- "loss": 0.024534843862056732,
- "num_tokens": 692541.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
"step": 169,
- "step_time": 3.9512340759993094
+ "step_time": 12.27045077799994
},
{
"clip_ratio/high_max": 0.0,
@@ -4747,27 +4747,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 67.875,
- "completions/mean_terminated_length": 67.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.2032299041748047,
- "epoch": 1.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11185794323682785,
- "kl": 1.5517784049734473e-05,
- "learning_rate": 5.264830667587295e-07,
- "loss": -0.05245225131511688,
- "num_tokens": 698064.0,
- "reward": 0.4444866180419922,
- "reward_std": 0.32400256395339966,
- "rewards/true_env_reward_fn/mean": 0.4444866180419922,
- "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
"step": 170,
- "step_time": 4.471538110999973
+ "step_time": 31.721865252000498
},
{
"clip_ratio/high_max": 0.0,
@@ -4776,26 +4776,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 95.0,
- "completions/mean_terminated_length": 95.0,
- "completions/min_length": 79.0,
- "completions/min_terminated_length": 79.0,
- "entropy": 1.4204387068748474,
- "epoch": 1.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1646534949541092,
- "kl": 2.4697198568901513e-05,
- "learning_rate": 5.228866172636899e-07,
- "loss": 0.02632315456867218,
- "num_tokens": 704196.0,
- "reward": 0.304565966129303,
- "reward_std": 0.32997164130210876,
- "rewards/true_env_reward_fn/mean": 0.304565966129303,
- "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
"step": 171,
- "step_time": 5.0436168590003945
+ "step_time": 8.973740739000277
},
{
"clip_ratio/high_max": 0.0,
@@ -4804,26 +4804,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1254178285598755,
- "epoch": 1.3983739837398375,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15058821439743042,
- "kl": 1.8407325114822015e-05,
- "learning_rate": 5.192791644186662e-07,
- "loss": 0.025478817522525787,
- "num_tokens": 706411.0,
- "reward": 0.7279239892959595,
- "reward_std": 0.11376125365495682,
- "rewards/true_env_reward_fn/mean": 0.7279239892959595,
- "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
"step": 172,
- "step_time": 3.2261944119982218
+ "step_time": 10.39117795600032
},
{
"clip_ratio/high_max": 0.0,
@@ -4832,26 +4832,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0894773602485657,
- "epoch": 1.4065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1687694787979126,
- "kl": 2.6046765015053097e-05,
- "learning_rate": 5.156610312374013e-07,
- "loss": -0.056941211223602295,
- "num_tokens": 711212.0,
- "reward": 0.4907146692276001,
- "reward_std": 0.3376546800136566,
- "rewards/true_env_reward_fn/mean": 0.4907146692276001,
- "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
"step": 173,
- "step_time": 3.9498180619993946
+ "step_time": 11.8111169230001
},
{
"clip_ratio/high_max": 0.0,
@@ -4860,26 +4860,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 60.125,
- "completions/mean_terminated_length": 60.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2238691449165344,
- "epoch": 1.4146341463414633,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19584441184997559,
- "kl": 4.8285241064149886e-05,
- "learning_rate": 5.120325416899629e-07,
- "loss": 0.0766875222325325,
- "num_tokens": 715409.0,
- "reward": 0.4593355059623718,
- "reward_std": 0.3909546136856079,
- "rewards/true_env_reward_fn/mean": 0.4593355059623718,
- "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
"step": 174,
- "step_time": 4.100519798999812
+ "step_time": 17.211099596999702
},
{
"clip_ratio/high_max": 0.0,
@@ -4888,26 +4888,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 60.25,
- "completions/mean_terminated_length": 60.25,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1763202548027039,
- "epoch": 1.4227642276422765,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1448555737733841,
- "kl": 1.2618989785551094e-05,
- "learning_rate": 5.08394020673734e-07,
- "loss": -0.012558378279209137,
- "num_tokens": 722327.0,
- "reward": 0.15966665744781494,
- "reward_std": 0.3235519230365753,
- "rewards/true_env_reward_fn/mean": 0.15966665744781494,
- "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
"step": 175,
- "step_time": 3.875348296000084
+ "step_time": 12.29185969800028
},
{
"clip_ratio/high_max": 0.0,
@@ -4916,26 +4916,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2852763533592224,
- "epoch": 1.4308943089430894,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.22303813695907593,
- "kl": 4.580334098136518e-05,
- "learning_rate": 5.047457939843227e-07,
- "loss": -0.09214464575052261,
- "num_tokens": 726828.0,
- "reward": 0.25830498337745667,
- "reward_std": 0.37860655784606934,
- "rewards/true_env_reward_fn/mean": 0.25830498337745667,
- "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
"step": 176,
- "step_time": 4.241473076999682
+ "step_time": 10.987576109999736
},
{
"clip_ratio/high_max": 0.0,
@@ -4944,26 +4944,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.3609731197357178,
- "epoch": 1.4390243902439024,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.001985745271668e-05,
- "kl": 1.4942165307729738e-05,
- "learning_rate": 5.010881882863893e-07,
- "loss": 7.44550789022469e-07,
- "num_tokens": 729930.0,
- "reward": 0.6203632950782776,
- "reward_std": 0.11898252367973328,
- "rewards/true_env_reward_fn/mean": 0.6203632950782776,
- "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
"step": 177,
- "step_time": 3.3902666960002534
+ "step_time": 10.792315139000493
},
{
"clip_ratio/high_max": 0.0,
@@ -4972,26 +4972,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2678966522216797,
- "epoch": 1.4471544715447155,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15351757407188416,
- "kl": 1.6737100395403104e-05,
- "learning_rate": 4.974215310843967e-07,
- "loss": 0.041131969541311264,
- "num_tokens": 732155.0,
- "reward": 0.7803820371627808,
- "reward_std": 0.08667682856321335,
- "rewards/true_env_reward_fn/mean": 0.7803820371627808,
- "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
"step": 178,
- "step_time": 3.637111981999624
+ "step_time": 12.399353334999887
},
{
"clip_ratio/high_max": 0.0,
@@ -5000,26 +5000,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.146271526813507,
- "epoch": 1.4552845528455285,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18466196954250336,
- "kl": 3.719841197380447e-05,
- "learning_rate": 4.937461506932859e-07,
- "loss": 0.029051154851913452,
- "num_tokens": 735418.0,
- "reward": 0.40377071499824524,
- "reward_std": 0.28345924615859985,
- "rewards/true_env_reward_fn/mean": 0.40377071499824524,
- "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
"step": 179,
- "step_time": 3.340555791999577
+ "step_time": 12.45691162600042
},
{
"clip_ratio/high_max": 0.0,
@@ -5028,26 +5028,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 170.0,
- "completions/max_terminated_length": 170.0,
- "completions/mean_length": 79.75,
- "completions/mean_terminated_length": 79.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.3902945518493652,
- "epoch": 1.4634146341463414,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1020251139998436,
- "kl": 1.8220500351162627e-05,
- "learning_rate": 4.900623762090777e-07,
- "loss": -0.002344265580177307,
- "num_tokens": 740540.0,
- "reward": 0.3590222895145416,
- "reward_std": 0.12487777322530746,
- "rewards/true_env_reward_fn/mean": 0.3590222895145416,
- "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
"step": 180,
- "step_time": 7.219923718001155
+ "step_time": 12.108760526000424
},
{
"clip_ratio/high_max": 0.0,
@@ -5056,26 +5056,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 60.875,
- "completions/mean_terminated_length": 60.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.517557680606842,
- "epoch": 1.4715447154471546,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24192294478416443,
- "kl": 4.1268089262302965e-05,
- "learning_rate": 4.863705374794055e-07,
- "loss": 0.09132950007915497,
- "num_tokens": 744723.0,
- "reward": 0.23991000652313232,
- "reward_std": 0.2837013602256775,
- "rewards/true_env_reward_fn/mean": 0.23991000652313232,
- "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
"step": 181,
- "step_time": 3.696339096999509
+ "step_time": 14.16400909000049
},
{
"clip_ratio/high_max": 0.0,
@@ -5084,26 +5084,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.249614655971527,
- "epoch": 1.4796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1566745787858963,
- "kl": 2.6629099920683075e-05,
- "learning_rate": 4.826709650739811e-07,
- "loss": 0.003972277045249939,
- "num_tokens": 748979.0,
- "reward": 0.4935140311717987,
- "reward_std": 0.41978561878204346,
- "rewards/true_env_reward_fn/mean": 0.4935140311717987,
- "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
"step": 182,
- "step_time": 3.316512920000605
+ "step_time": 19.767916835999586
},
{
"clip_ratio/high_max": 0.0,
@@ -5112,26 +5112,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1666916608810425,
- "epoch": 1.4878048780487805,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001005820304271765,
- "kl": 1.2246940059412736e-05,
- "learning_rate": 4.789639902549948e-07,
- "loss": 6.330609494398232e-07,
- "num_tokens": 751323.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
"step": 183,
- "step_time": 3.7099916660008603
+ "step_time": 10.428195530999346
},
{
"clip_ratio/high_max": 0.0,
@@ -5140,26 +5140,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.3246222138404846,
- "epoch": 1.4959349593495934,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.18140996992588043,
- "kl": 3.1042441150930244e-05,
- "learning_rate": 4.752499449474535e-07,
- "loss": -0.022353097796440125,
- "num_tokens": 755494.0,
- "reward": 0.4561777412891388,
- "reward_std": 0.2439236342906952,
- "rewards/true_env_reward_fn/mean": 0.4561777412891388,
- "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
"step": 184,
- "step_time": 3.7916486710000754
+ "step_time": 16.47171987999991
},
{
"clip_ratio/high_max": 0.0,
@@ -5168,26 +5168,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 75.75,
- "completions/mean_terminated_length": 75.75,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.2625707983970642,
- "epoch": 1.5040650406504064,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11702944338321686,
- "kl": 1.503958355897339e-05,
- "learning_rate": 4.715291617094607e-07,
- "loss": 0.023916304111480713,
- "num_tokens": 758432.0,
- "reward": 0.5015827417373657,
- "reward_std": 0.1783808320760727,
- "rewards/true_env_reward_fn/mean": 0.5015827417373657,
- "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
"step": 185,
- "step_time": 4.226409274000616
+ "step_time": 13.684267182999974
},
{
"clip_ratio/high_max": 0.0,
@@ -5196,26 +5196,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 54.0,
- "completions/mean_terminated_length": 54.0,
- "completions/min_length": 25.0,
- "completions/min_terminated_length": 25.0,
- "entropy": 1.1460023522377014,
- "epoch": 1.5121951219512195,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11851620674133301,
- "kl": 1.3728345948038623e-05,
- "learning_rate": 4.678019737024387e-07,
- "loss": 0.0831337422132492,
- "num_tokens": 764336.0,
- "reward": 0.19094166159629822,
- "reward_std": 0.30934420228004456,
- "rewards/true_env_reward_fn/mean": 0.19094166159629822,
- "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
"step": 186,
- "step_time": 3.563357556000483
+ "step_time": 10.281018189999031
},
{
"clip_ratio/high_max": 0.0,
@@ -5224,26 +5224,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1291148662567139,
- "epoch": 1.5203252032520327,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.864256960805506e-05,
- "kl": 1.081683785741916e-05,
- "learning_rate": 4.6406871466129704e-07,
- "loss": 5.412177870312007e-07,
- "num_tokens": 766608.0,
- "reward": 0.8357743620872498,
- "reward_std": 0.11130158603191376,
- "rewards/true_env_reward_fn/mean": 0.8357743620872498,
- "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
"step": 187,
- "step_time": 3.534869859002356
+ "step_time": 24.351223329000277
},
{
"clip_ratio/high_max": 0.0,
@@ -5252,26 +5252,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 189.0,
- "completions/max_terminated_length": 189.0,
- "completions/mean_length": 92.0,
- "completions/mean_terminated_length": 92.0,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.3645328283309937,
- "epoch": 1.5284552845528454,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18168245255947113,
- "kl": 2.6857565899263136e-05,
- "learning_rate": 4.6032971886454956e-07,
- "loss": -0.005156125873327255,
- "num_tokens": 774496.0,
- "reward": 0.11249999701976776,
- "reward_std": 0.20856082439422607,
- "rewards/true_env_reward_fn/mean": 0.11249999701976776,
- "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
"step": 188,
- "step_time": 8.34005261099992
+ "step_time": 10.644911742000204
},
{
"clip_ratio/high_max": 0.0,
@@ -5280,26 +5280,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1495982110500336,
- "epoch": 1.5365853658536586,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14486820995807648,
- "kl": 1.3202762147557223e-05,
- "learning_rate": 4.5658532110438337e-07,
- "loss": -0.0010610297322273254,
- "num_tokens": 777186.0,
- "reward": 0.5879127383232117,
- "reward_std": 0.05142820253968239,
- "rewards/true_env_reward_fn/mean": 0.5879127383232117,
- "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
"step": 189,
- "step_time": 3.4856022139993
+ "step_time": 9.579594637999435
},
{
"clip_ratio/high_max": 0.0,
@@ -5308,26 +5308,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.241140365600586,
- "epoch": 1.5447154471544715,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001250067143701017,
- "kl": 1.5482702110602986e-05,
- "learning_rate": 4.52835856656681e-07,
- "loss": 7.80837922320643e-07,
- "num_tokens": 779965.0,
- "reward": 0.6861198544502258,
- "reward_std": 0.08807206153869629,
- "rewards/true_env_reward_fn/mean": 0.6861198544502258,
- "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
"step": 190,
- "step_time": 3.904181735999373
+ "step_time": 11.495368679999501
},
{
"clip_ratio/high_max": 0.0,
@@ -5336,26 +5336,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2944807410240173,
- "epoch": 1.5528455284552845,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2030111849308014,
- "kl": 3.585523518268019e-05,
- "learning_rate": 4.490816612509991e-07,
- "loss": 0.0143373291939497,
- "num_tokens": 786140.0,
- "reward": 0.35173332691192627,
- "reward_std": 0.18115806579589844,
- "rewards/true_env_reward_fn/mean": 0.35173332691192627,
- "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
"step": 191,
- "step_time": 4.005758510000305
+ "step_time": 7.909104242000012
},
{
"clip_ratio/high_max": 0.0,
@@ -5364,26 +5364,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 58.0,
- "completions/mean_terminated_length": 58.0,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2177271246910095,
- "epoch": 1.5609756097560976,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11188288033008575,
- "kl": 2.3622495064046234e-05,
- "learning_rate": 4.45323071040508e-07,
- "loss": -0.02369789034128189,
- "num_tokens": 790424.0,
- "reward": 0.4888629913330078,
- "reward_std": 0.23310808837413788,
- "rewards/true_env_reward_fn/mean": 0.4888629913330078,
- "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
"step": 192,
- "step_time": 3.2518814809991454
+ "step_time": 10.043768619000275
},
{
"clip_ratio/high_max": 0.0,
@@ -5392,26 +5392,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.192966103553772,
- "epoch": 1.5691056910569106,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1378823071718216,
- "kl": 1.7358055174554465e-05,
- "learning_rate": 4.4156042257189143e-07,
- "loss": 0.06256310641765594,
- "num_tokens": 794521.0,
- "reward": 0.5219699740409851,
- "reward_std": 0.06214587390422821,
- "rewards/true_env_reward_fn/mean": 0.5219699740409851,
- "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
"step": 193,
- "step_time": 4.222739491999164
+ "step_time": 10.19648474899941
},
{
"clip_ratio/high_max": 0.0,
@@ -5420,26 +5420,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 63.125,
- "completions/mean_terminated_length": 63.125,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1770159006118774,
- "epoch": 1.5772357723577235,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19550059735774994,
- "kl": 2.4871268578863237e-05,
- "learning_rate": 4.377940527552125e-07,
- "loss": 0.05841376632452011,
- "num_tokens": 798194.0,
- "reward": 0.42302167415618896,
- "reward_std": 0.2911272346973419,
- "rewards/true_env_reward_fn/mean": 0.42302167415618896,
- "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
"step": 194,
- "step_time": 3.8170270639984665
+ "step_time": 14.372816425000565
},
{
"clip_ratio/high_max": 0.0,
@@ -5448,26 +5448,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.5244255661964417,
- "epoch": 1.5853658536585367,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1872202306985855,
- "kl": 2.5990483663917985e-05,
- "learning_rate": 4.340242988337462e-07,
- "loss": -0.044112429022789,
- "num_tokens": 802802.0,
- "reward": 0.33745431900024414,
- "reward_std": 0.22955451905727386,
- "rewards/true_env_reward_fn/mean": 0.33745431900024414,
- "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
"step": 195,
- "step_time": 4.27381555500142
+ "step_time": 11.282298853000157
},
{
"clip_ratio/high_max": 0.0,
@@ -5476,26 +5476,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.345891296863556,
- "epoch": 1.5934959349593496,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16891054809093475,
- "kl": 2.7261638024356216e-05,
- "learning_rate": 4.3025149835378275e-07,
- "loss": -0.139386385679245,
- "num_tokens": 807881.0,
- "reward": 0.3240283131599426,
- "reward_std": 0.2803676128387451,
- "rewards/true_env_reward_fn/mean": 0.3240283131599426,
- "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
"step": 196,
- "step_time": 4.981287381999209
+ "step_time": 8.65745804199969
},
{
"clip_ratio/high_max": 0.0,
@@ -5504,26 +5504,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.246802031993866,
- "epoch": 1.6016260162601625,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2096729576587677,
- "kl": 3.5958016269432846e-05,
- "learning_rate": 4.2647598913440264e-07,
- "loss": -0.02941281348466873,
- "num_tokens": 812500.0,
- "reward": 0.4126526415348053,
- "reward_std": 0.36393746733665466,
- "rewards/true_env_reward_fn/mean": 0.4126526415348053,
- "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
"step": 197,
- "step_time": 3.101726017999681
+ "step_time": 9.615110594999805
},
{
"clip_ratio/high_max": 0.0,
@@ -5532,26 +5532,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4369062185287476,
- "epoch": 1.6097560975609757,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1862105429172516,
- "kl": 4.7646244638599455e-05,
- "learning_rate": 4.2269810923722965e-07,
- "loss": 0.0521145761013031,
- "num_tokens": 818718.0,
- "reward": 0.26869943737983704,
- "reward_std": 0.1483483463525772,
- "rewards/true_env_reward_fn/mean": 0.26869943737983704,
- "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
"step": 198,
- "step_time": 4.287780451000799
+ "step_time": 10.832162847999825
},
{
"clip_ratio/high_max": 0.0,
@@ -5560,26 +5560,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 21.0,
- "completions/min_terminated_length": 21.0,
- "entropy": 1.192937195301056,
- "epoch": 1.6178861788617886,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2013663798570633,
- "kl": 1.8760739294521045e-05,
- "learning_rate": 4.189181969361588e-07,
- "loss": 0.07236722111701965,
- "num_tokens": 825728.0,
- "reward": 0.23110000789165497,
- "reward_std": 0.23212090134620667,
- "rewards/true_env_reward_fn/mean": 0.23110000789165497,
- "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
"step": 199,
- "step_time": 4.708717262998107
+ "step_time": 10.94759418800004
},
{
"clip_ratio/high_max": 0.0,
@@ -5588,26 +5588,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 72.25,
- "completions/mean_terminated_length": 72.25,
- "completions/min_length": 62.0,
- "completions/min_terminated_length": 62.0,
- "entropy": 1.3792839050292969,
- "epoch": 1.6260162601626016,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.7985117463395e-05,
- "kl": 1.4280476534622721e-05,
- "learning_rate": 4.1513659068706814e-07,
- "loss": 7.153485626076872e-07,
- "num_tokens": 830318.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
"step": 200,
- "step_time": 3.9055351140013954
+ "step_time": 10.322844021000492
},
{
"clip_ratio/high_max": 0.0,
@@ -5616,54 +5616,54 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.0505937337875366,
- "epoch": 1.6341463414634148,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13188514113426208,
- "kl": 2.842090088961413e-05,
- "learning_rate": 4.1135362909751326e-07,
- "loss": -0.0017508119344711304,
- "num_tokens": 834933.0,
- "reward": 0.36545002460479736,
- "reward_std": 0.24526984989643097,
- "rewards/true_env_reward_fn/mean": 0.36545002460479736,
- "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.083335876464844,
+ "completions/mean_terminated_length": 62.083335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.319727510213852,
+ "epoch": 4.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08774503320455551,
+ "kl": 6.86226876496221e-05,
+ "learning_rate": 6.077002463994907e-07,
+ "loss": 0.058894164860248566,
+ "num_tokens": 4956466.0,
+ "reward": 0.41384777426719666,
+ "reward_std": 0.17193447053432465,
+ "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+ "rewards/true_env_reward_fn/std": 0.17193445563316345,
"step": 201,
- "step_time": 3.895525625997834
+ "step_time": 10.146928047999609
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 163.0,
- "completions/max_terminated_length": 163.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.3750707507133484,
- "epoch": 1.6422764227642277,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1798955649137497,
- "kl": 2.5428611479583196e-05,
- "learning_rate": 4.075696508964076e-07,
- "loss": 0.1843666434288025,
- "num_tokens": 838076.0,
- "reward": 0.44641831517219543,
- "reward_std": 0.30040720105171204,
- "rewards/true_env_reward_fn/mean": 0.44641831517219543,
- "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 59.395835876464844,
+ "completions/mean_terminated_length": 59.395835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1935442388057709,
+ "epoch": 4.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08328510820865631,
+ "kl": 7.948942857183283e-05,
+ "learning_rate": 6.035394200708103e-07,
+ "loss": 0.045964501798152924,
+ "num_tokens": 4978733.0,
+ "reward": 0.4896667003631592,
+ "reward_std": 0.3830767571926117,
+ "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+ "rewards/true_env_reward_fn/std": 0.3830767571926117,
"step": 202,
- "step_time": 6.74463491500137
+ "step_time": 10.025533761000133
},
{
"clip_ratio/high_max": 0.0,
@@ -5672,26 +5672,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 132.0,
- "completions/max_terminated_length": 132.0,
- "completions/mean_length": 79.125,
- "completions/mean_terminated_length": 79.125,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3369249105453491,
- "epoch": 1.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11249764263629913,
- "kl": 1.1453126717242412e-05,
- "learning_rate": 4.0378499490369267e-07,
- "loss": -0.08210685849189758,
- "num_tokens": 841933.0,
- "reward": 0.3737962245941162,
- "reward_std": 0.13184049725532532,
- "rewards/true_env_reward_fn/mean": 0.3737962245941162,
- "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 62.22916793823242,
+ "completions/mean_terminated_length": 62.22916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2698631286621094,
+ "epoch": 4.951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07158155739307404,
+ "kl": 8.286665615742095e-05,
+ "learning_rate": 5.993710887521302e-07,
+ "loss": -0.03551984950900078,
+ "num_tokens": 5009440.0,
+ "reward": 0.45639634132385254,
+ "reward_std": 0.1837630569934845,
+ "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+ "rewards/true_env_reward_fn/std": 0.1837630420923233,
"step": 203,
- "step_time": 5.5639925510004105
+ "step_time": 9.823523802999716
},
{
"clip_ratio/high_max": 0.0,
@@ -5700,26 +5700,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.5737199783325195,
- "epoch": 1.6585365853658538,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19394491612911224,
- "kl": 5.05317857459886e-05,
- "learning_rate": 4e-07,
- "loss": 0.05622926354408264,
- "num_tokens": 847711.0,
- "reward": 0.10725000500679016,
- "reward_std": 0.19980257749557495,
- "rewards/true_env_reward_fn/mean": 0.10725000500679016,
- "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "entropy": 1.2825455367565155,
+ "epoch": 4.975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07431793212890625,
+ "kl": 7.247529447340639e-05,
+ "learning_rate": 5.951955545823342e-07,
+ "loss": -0.02418600022792816,
+ "num_tokens": 5036036.0,
+ "reward": 0.4476773142814636,
+ "reward_std": 0.20447416603565216,
+ "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+ "rewards/true_env_reward_fn/std": 0.20447418093681335,
"step": 204,
- "step_time": 4.918089437000162
+ "step_time": 11.960790695000014
},
{
"clip_ratio/high_max": 0.0,
@@ -5728,26 +5728,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 60.5,
- "completions/mean_terminated_length": 60.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.119917094707489,
- "epoch": 1.6666666666666665,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10553985089063644,
- "kl": 1.2793303994840244e-05,
- "learning_rate": 3.9621500509630725e-07,
- "loss": -0.011355768889188766,
- "num_tokens": 849519.0,
- "reward": 0.6593211889266968,
- "reward_std": 0.11862105131149292,
- "rewards/true_env_reward_fn/mean": 0.6593211889266968,
- "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2438389956951141,
+ "epoch": 5.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06833480298519135,
+ "kl": 5.646793260893901e-05,
+ "learning_rate": 5.91013120222401e-07,
+ "loss": 0.0021926667541265488,
+ "num_tokens": 5063228.0,
+ "reward": 0.4300723075866699,
+ "reward_std": 0.12116922438144684,
+ "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+ "rewards/true_env_reward_fn/std": 0.12116922438144684,
"step": 205,
- "step_time": 4.127652793999005
+ "step_time": 9.874485716000436
},
{
"clip_ratio/high_max": 0.0,
@@ -5756,26 +5756,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 108.0,
- "completions/max_terminated_length": 108.0,
- "completions/mean_length": 75.125,
- "completions/mean_terminated_length": 75.125,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3856809735298157,
- "epoch": 1.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1911258101463318,
- "kl": 4.2569914512569085e-05,
- "learning_rate": 3.9243034910359247e-07,
- "loss": 0.10561336576938629,
- "num_tokens": 854156.0,
- "reward": 0.2516202926635742,
- "reward_std": 0.22357939183712006,
- "rewards/true_env_reward_fn/mean": 0.2516202926635742,
- "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "completions/max_length": 205.0,
+ "completions/max_terminated_length": 205.0,
+ "completions/mean_length": 66.1875,
+ "completions/mean_terminated_length": 66.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2645181119441986,
+ "epoch": 5.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08248262107372284,
+ "kl": 9.333990783488844e-05,
+ "learning_rate": 5.868240888334652e-07,
+ "loss": 0.12253005057573318,
+ "num_tokens": 5093373.0,
+ "reward": 0.28542038798332214,
+ "reward_std": 0.326623797416687,
+ "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+ "rewards/true_env_reward_fn/std": 0.326623797416687,
"step": 206,
- "step_time": 4.714620994000143
+ "step_time": 15.706792760000553
},
{
"clip_ratio/high_max": 0.0,
@@ -5784,26 +5784,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.4012945890426636,
- "epoch": 1.6829268292682928,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226125180721283,
- "kl": 5.4918069963605376e-05,
- "learning_rate": 3.886463709024868e-07,
- "loss": 0.012949362397193909,
- "num_tokens": 856151.0,
- "reward": 0.6204804182052612,
- "reward_std": 0.44673967361450195,
- "rewards/true_env_reward_fn/mean": 0.6204804182052612,
- "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.16667175292969,
+ "completions/mean_terminated_length": 67.16667175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.34083291888237,
+ "epoch": 5.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07931552827358246,
+ "kl": 0.00010715152166085318,
+ "learning_rate": 5.826287640548424e-07,
+ "loss": 0.0005592256784439087,
+ "num_tokens": 5118933.0,
+ "reward": 0.5035215616226196,
+ "reward_std": 0.304157018661499,
+ "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+ "rewards/true_env_reward_fn/std": 0.30415698885917664,
"step": 207,
- "step_time": 4.094810713999323
+ "step_time": 10.04168460900064
},
{
"clip_ratio/high_max": 0.0,
@@ -5812,26 +5812,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.3708943128585815,
- "epoch": 1.6910569105691056,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12903675436973572,
- "kl": 1.4145812656352064e-05,
- "learning_rate": 3.8486340931293187e-07,
- "loss": -0.07838249206542969,
- "num_tokens": 859380.0,
- "reward": 0.6400156021118164,
- "reward_std": 0.1022576317191124,
- "rewards/true_env_reward_fn/mean": 0.6400156021118164,
- "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 73.4375,
+ "completions/mean_terminated_length": 73.4375,
+ "completions/min_length": 11.0,
+ "completions/min_terminated_length": 11.0,
+ "entropy": 1.1821868121623993,
+ "epoch": 5.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07619761675596237,
+ "kl": 0.00013189401397539768,
+ "learning_rate": 5.784274499820213e-07,
+ "loss": -0.011967688798904419,
+ "num_tokens": 5148682.0,
+ "reward": 0.36097532510757446,
+ "reward_std": 0.29208436608314514,
+ "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+ "rewards/true_env_reward_fn/std": 0.29208436608314514,
"step": 208,
- "step_time": 4.297900428997309
+ "step_time": 15.226898961000188
},
{
"clip_ratio/high_max": 0.0,
@@ -5840,26 +5840,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.75,
- "completions/mean_terminated_length": 56.75,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2782961130142212,
- "epoch": 1.6991869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12437833100557327,
- "kl": 1.4473939245362999e-05,
- "learning_rate": 3.8108180306384135e-07,
- "loss": -0.036324724555015564,
- "num_tokens": 862714.0,
- "reward": 0.43419933319091797,
- "reward_std": 0.15345513820648193,
- "rewards/true_env_reward_fn/mean": 0.43419933319091797,
- "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 75.39583587646484,
+ "completions/mean_terminated_length": 75.39583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2636725008487701,
+ "epoch": 5.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06586338579654694,
+ "kl": 1.9426477138040354e-05,
+ "learning_rate": 5.742204511446203e-07,
+ "loss": -0.047095417976379395,
+ "num_tokens": 5173205.0,
+ "reward": 0.5261925458908081,
+ "reward_std": 0.2689943313598633,
+ "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+ "rewards/true_env_reward_fn/std": 0.2689943313598633,
"step": 209,
- "step_time": 3.47861851900052
+ "step_time": 23.690397457000017
},
{
"clip_ratio/high_max": 0.0,
@@ -5867,27 +5867,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 512.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 123.875,
- "completions/mean_terminated_length": 68.42857360839844,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.5170292258262634,
- "epoch": 1.7073170731707317,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 54.3125,
+ "completions/mean_terminated_length": 54.3125,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2415392696857452,
+ "epoch": 5.121951219512195,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1543467938899994,
- "kl": 2.3121557205740828e-05,
- "learning_rate": 3.7730189076277037e-07,
- "loss": -0.3810324966907501,
- "num_tokens": 869789.0,
- "reward": 0.3795333504676819,
- "reward_std": 0.13815106451511383,
- "rewards/true_env_reward_fn/mean": 0.3795333504676819,
- "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "grad_norm": 0.0990133062005043,
+ "kl": 7.119746442185715e-05,
+ "learning_rate": 5.700080724843146e-07,
+ "loss": -0.030588299036026,
+ "num_tokens": 5199892.0,
+ "reward": 0.3418487012386322,
+ "reward_std": 0.36353805661201477,
+ "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+ "rewards/true_env_reward_fn/std": 0.3635380268096924,
"step": 210,
- "step_time": 20.698896928999602
+ "step_time": 9.864614251999683
},
{
"clip_ratio/high_max": 0.0,
@@ -5896,26 +5896,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 69.875,
- "completions/mean_terminated_length": 69.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2324861884117126,
- "epoch": 1.7154471544715446,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11762262135744095,
- "kl": 1.9743249595194357e-05,
- "learning_rate": 3.735240108655973e-07,
- "loss": -0.09104303270578384,
- "num_tokens": 872604.0,
- "reward": 0.5916227102279663,
- "reward_std": 0.1752101182937622,
- "rewards/true_env_reward_fn/mean": 0.5916227102279663,
- "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2508135735988617,
+ "epoch": 5.146341463414634,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10835102945566177,
+ "kl": 0.00012395972225931473,
+ "learning_rate": 5.657906193327324e-07,
+ "loss": 0.08621911704540253,
+ "num_tokens": 5220141.0,
+ "reward": 0.4987506866455078,
+ "reward_std": 0.27843451499938965,
+ "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+ "rewards/true_env_reward_fn/std": 0.27843451499938965,
"step": 211,
- "step_time": 5.218213289999767
+ "step_time": 9.983622502000344
},
{
"clip_ratio/high_max": 0.0,
@@ -5924,26 +5924,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.4448966979980469,
- "epoch": 1.7235772357723578,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1924143135547638,
- "kl": 3.186432604707079e-05,
- "learning_rate": 3.697485016462174e-07,
- "loss": 0.025449808686971664,
- "num_tokens": 877921.0,
- "reward": 0.1028124988079071,
- "reward_std": 0.2254277467727661,
- "rewards/true_env_reward_fn/mean": 0.1028124988079071,
- "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1611086130142212,
+ "epoch": 5.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08423946797847748,
+ "kl": 4.4347343191475375e-05,
+ "learning_rate": 5.615683973893234e-07,
+ "loss": 0.08670675754547119,
+ "num_tokens": 5244470.0,
+ "reward": 0.4252437949180603,
+ "reward_std": 0.2996494472026825,
+ "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+ "rewards/true_env_reward_fn/std": 0.2996494472026825,
"step": 212,
- "step_time": 3.415528882000217
+ "step_time": 18.68646409699977
},
{
"clip_ratio/high_max": 0.0,
@@ -5952,26 +5952,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 59.875,
- "completions/mean_terminated_length": 59.875,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
"completions/min_length": 32.0,
"completions/min_terminated_length": 32.0,
- "entropy": 1.185910701751709,
- "epoch": 1.7317073170731707,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001214127623825334,
- "kl": 1.3199866316426778e-05,
- "learning_rate": 3.659757011662538e-07,
- "loss": 6.776077725589857e-07,
- "num_tokens": 880344.0,
- "reward": 0.7329437732696533,
- "reward_std": 0.22123214602470398,
- "rewards/true_env_reward_fn/mean": 0.7329437732696533,
- "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "entropy": 1.3110275566577911,
+ "epoch": 5.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06817487627267838,
+ "kl": 0.00010393545744591393,
+ "learning_rate": 5.573417126992002e-07,
+ "loss": 0.04062439873814583,
+ "num_tokens": 5271369.0,
+ "reward": 0.4340522885322571,
+ "reward_std": 0.26940545439720154,
+ "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+ "rewards/true_env_reward_fn/std": 0.26940542459487915,
"step": 213,
- "step_time": 3.5156538789997285
+ "step_time": 11.3590317649996
},
{
"clip_ratio/high_max": 0.0,
@@ -5980,26 +5980,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 162.0,
- "completions/max_terminated_length": 162.0,
- "completions/mean_length": 90.75,
- "completions/mean_terminated_length": 90.75,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2697569727897644,
- "epoch": 1.7398373983739837,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 54.4375,
+ "completions/mean_terminated_length": 54.4375,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2018881738185883,
+ "epoch": 5.219512195121951,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1553589552640915,
- "kl": 1.621047795197228e-05,
- "learning_rate": 3.622059472447875e-07,
- "loss": -0.111361563205719,
- "num_tokens": 885006.0,
- "reward": 0.5490846633911133,
- "reward_std": 0.14710450172424316,
- "rewards/true_env_reward_fn/mean": 0.5490846633911133,
- "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "grad_norm": 0.10059589147567749,
+ "kl": 0.00011373830420779996,
+ "learning_rate": 5.531108716309547e-07,
+ "loss": -0.09816272556781769,
+ "num_tokens": 5288638.0,
+ "reward": 0.5691710710525513,
+ "reward_std": 0.25253745913505554,
+ "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+ "rewards/true_env_reward_fn/std": 0.25253745913505554,
"step": 214,
- "step_time": 7.051423932000034
+ "step_time": 9.633293675000004
},
{
"clip_ratio/high_max": 0.0,
@@ -6008,26 +6008,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.1904898285865784,
- "epoch": 1.7479674796747968,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19009968638420105,
- "kl": 1.6167180092452327e-05,
- "learning_rate": 3.5843957742810864e-07,
- "loss": -0.006048411130905151,
- "num_tokens": 888255.0,
- "reward": 0.5893601179122925,
- "reward_std": 0.16021940112113953,
- "rewards/true_env_reward_fn/mean": 0.5893601179122925,
- "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1958912312984467,
+ "epoch": 5.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0779944583773613,
+ "kl": 7.025236300250981e-05,
+ "learning_rate": 5.488761808544509e-07,
+ "loss": 0.03371567651629448,
+ "num_tokens": 5312081.0,
+ "reward": 0.44519662857055664,
+ "reward_std": 0.22201679646968842,
+ "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+ "rewards/true_env_reward_fn/std": 0.22201678156852722,
"step": 215,
- "step_time": 3.0270869319992926
+ "step_time": 12.463993090999338
},
{
"clip_ratio/high_max": 0.0,
@@ -6036,26 +6036,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.2750649452209473,
- "epoch": 1.7560975609756098,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1521027386188507,
- "kl": 4.317680577514693e-05,
- "learning_rate": 3.5467692895949205e-07,
- "loss": -0.04247616231441498,
- "num_tokens": 892490.0,
- "reward": 0.1841849982738495,
- "reward_std": 0.30015870928764343,
- "rewards/true_env_reward_fn/mean": 0.1841849982738495,
- "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.2146256864070892,
+ "epoch": 5.2682926829268295,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06834157556295395,
+ "kl": 0.00010125362859980669,
+ "learning_rate": 5.446379473185971e-07,
+ "loss": -0.02198379673063755,
+ "num_tokens": 5334238.0,
+ "reward": 0.5273472666740417,
+ "reward_std": 0.21954773366451263,
+ "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+ "rewards/true_env_reward_fn/std": 0.21954771876335144,
"step": 216,
- "step_time": 3.461749838998003
+ "step_time": 15.126136884000061
},
{
"clip_ratio/high_max": 0.0,
@@ -6064,26 +6064,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.25,
- "completions/mean_terminated_length": 62.25,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2730335593223572,
- "epoch": 1.7642276422764227,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11831019073724747,
- "kl": 2.1095927877468057e-05,
- "learning_rate": 3.509183387490009e-07,
- "loss": -0.051231447607278824,
- "num_tokens": 894688.0,
- "reward": 0.49520131945610046,
- "reward_std": 0.3804744482040405,
- "rewards/true_env_reward_fn/mean": 0.49520131945610046,
- "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 61.333335876464844,
+ "completions/mean_terminated_length": 61.333335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2872387170791626,
+ "epoch": 5.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09578828513622284,
+ "kl": 7.404103780572768e-05,
+ "learning_rate": 5.403964782290961e-07,
+ "loss": -0.041091397404670715,
+ "num_tokens": 5351798.0,
+ "reward": 0.5464547872543335,
+ "reward_std": 0.16224616765975952,
+ "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+ "rewards/true_env_reward_fn/std": 0.16224615275859833,
"step": 217,
- "step_time": 3.7184635590019752
+ "step_time": 7.2901647220001
},
{
"clip_ratio/high_max": 0.0,
@@ -6092,26 +6092,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 47.75,
- "completions/mean_terminated_length": 47.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3015827536582947,
- "epoch": 1.7723577235772359,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21724702417850494,
- "kl": 3.562447545846226e-05,
- "learning_rate": 3.471641433433191e-07,
- "loss": -0.028775859624147415,
- "num_tokens": 899066.0,
- "reward": 0.3376166820526123,
- "reward_std": 0.21976198256015778,
- "rewards/true_env_reward_fn/mean": 0.3376166820526123,
- "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 84.89583587646484,
+ "completions/mean_terminated_length": 84.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.4322342276573181,
+ "epoch": 5.317073170731708,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0667513981461525,
+ "kl": 2.497344303264981e-05,
+ "learning_rate": 5.361520810261778e-07,
+ "loss": -0.06392769515514374,
+ "num_tokens": 5375369.0,
+ "reward": 0.5213420391082764,
+ "reward_std": 0.1436246931552887,
+ "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+ "rewards/true_env_reward_fn/std": 0.1436246782541275,
"step": 218,
- "step_time": 3.344433074000335
+ "step_time": 12.217936152999755
},
{
"clip_ratio/high_max": 0.0,
@@ -6120,26 +6120,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 53.125,
- "completions/mean_terminated_length": 53.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1390373706817627,
- "epoch": 1.7804878048780488,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001077545020962134,
- "kl": 1.554161144667887e-05,
- "learning_rate": 3.434146788956166e-07,
- "loss": 7.676237032683275e-07,
- "num_tokens": 902263.0,
- "reward": 0.5682899951934814,
- "reward_std": 0.1217179074883461,
- "rewards/true_env_reward_fn/mean": 0.5682899951934814,
- "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2382279634475708,
+ "epoch": 5.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07372384518384933,
+ "kl": 9.740726909512887e-05,
+ "learning_rate": 5.319050633623141e-07,
+ "loss": 0.010912742465734482,
+ "num_tokens": 5397956.0,
+ "reward": 0.5623860359191895,
+ "reward_std": 0.285375714302063,
+ "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+ "rewards/true_env_reward_fn/std": 0.285375714302063,
"step": 219,
- "step_time": 3.0424018219982827
+ "step_time": 13.449634822999997
},
{
"clip_ratio/high_max": 0.0,
@@ -6148,26 +6148,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.875,
- "completions/mean_terminated_length": 61.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1991714239120483,
- "epoch": 1.7886178861788617,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11428863555192947,
- "kl": 1.1265870853094384e-05,
- "learning_rate": 3.3967028113545045e-07,
- "loss": -0.02927359938621521,
- "num_tokens": 906206.0,
- "reward": 0.4925253391265869,
- "reward_std": 0.10813789069652557,
- "rewards/true_env_reward_fn/mean": 0.4925253391265869,
- "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 65.39583587646484,
+ "completions/mean_terminated_length": 65.39583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2669419348239899,
+ "epoch": 5.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07886125147342682,
+ "kl": 0.00011165817522851285,
+ "learning_rate": 5.276557330799203e-07,
+ "loss": -0.02433733269572258,
+ "num_tokens": 5418511.0,
+ "reward": 0.6075118780136108,
+ "reward_std": 0.30748677253723145,
+ "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+ "rewards/true_env_reward_fn/std": 0.30748677253723145,
"step": 220,
- "step_time": 3.5531271640011255
+ "step_time": 12.051496982000117
},
{
"clip_ratio/high_max": 0.0,
@@ -6176,26 +6176,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2546668648719788,
- "epoch": 1.796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1504630595445633,
- "kl": 1.5663241811125772e-05,
- "learning_rate": 3.3593128533870314e-07,
- "loss": 0.04806854575872421,
- "num_tokens": 909149.0,
- "reward": 0.49599751830101013,
- "reward_std": 0.15429075062274933,
- "rewards/true_env_reward_fn/mean": 0.49599751830101013,
- "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2560602128505707,
+ "epoch": 5.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08607357740402222,
+ "kl": 8.424731549894204e-05,
+ "learning_rate": 5.234043981890393e-07,
+ "loss": 0.013634156435728073,
+ "num_tokens": 5439690.0,
+ "reward": 0.5080039501190186,
+ "reward_std": 0.21975299715995789,
+ "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+ "rewards/true_env_reward_fn/std": 0.21975299715995789,
"step": 221,
- "step_time": 3.1345955030010373
+ "step_time": 10.435893627000496
},
{
"clip_ratio/high_max": 0.0,
@@ -6204,26 +6204,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1428714394569397,
- "epoch": 1.8048780487804879,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.310600969707593e-05,
- "kl": 1.1706195891747484e-05,
- "learning_rate": 3.321980262975613e-07,
- "loss": 5.957842290627013e-07,
- "num_tokens": 914211.0,
- "reward": 0.3258306384086609,
- "reward_std": 0.4338511824607849,
- "rewards/true_env_reward_fn/mean": 0.3258306384086609,
- "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3081265985965729,
+ "epoch": 5.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0876006931066513,
+ "kl": 9.127605972025776e-05,
+ "learning_rate": 5.191513668450177e-07,
+ "loss": -0.01968565583229065,
+ "num_tokens": 5468216.0,
+ "reward": 0.3679848611354828,
+ "reward_std": 0.237859845161438,
+ "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+ "rewards/true_env_reward_fn/std": 0.237859845161438,
"step": 222,
- "step_time": 3.8445859539988305
+ "step_time": 11.524256381999294
},
{
"clip_ratio/high_max": 0.0,
@@ -6232,26 +6232,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.1629019975662231,
- "epoch": 1.8130081300813008,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010947151895379648,
- "kl": 1.3530024261854123e-05,
- "learning_rate": 3.2847083829053923e-07,
- "loss": 6.723923888785066e-07,
- "num_tokens": 916850.0,
- "reward": 0.5956059694290161,
- "reward_std": 0.09251586347818375,
- "rewards/true_env_reward_fn/mean": 0.5956059694290161,
- "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 67.66667175292969,
+ "completions/mean_terminated_length": 67.66667175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2990808486938477,
+ "epoch": 5.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07685395330190659,
+ "kl": 0.00014209141045284923,
+ "learning_rate": 5.148969473261679e-07,
+ "loss": -0.01107364147901535,
+ "num_tokens": 5488040.0,
+ "reward": 0.5435788035392761,
+ "reward_std": 0.35419100522994995,
+ "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+ "rewards/true_env_reward_fn/std": 0.35419100522994995,
"step": 223,
- "step_time": 3.3021794950000185
+ "step_time": 11.945272217000365
},
{
"clip_ratio/high_max": 0.0,
@@ -6260,26 +6260,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.1237311363220215,
- "epoch": 1.821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14958374202251434,
- "kl": 4.083753810846247e-05,
- "learning_rate": 3.2475005505254657e-07,
- "loss": 0.09789139032363892,
- "num_tokens": 919901.0,
- "reward": 0.4179220199584961,
- "reward_std": 0.2486819326877594,
- "rewards/true_env_reward_fn/mean": 0.4179220199584961,
- "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.284348964691162,
+ "epoch": 5.463414634146342,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.058678388595581055,
+ "kl": 3.3754420201148605e-05,
+ "learning_rate": 5.106414480114238e-07,
+ "loss": -0.01714605651795864,
+ "num_tokens": 5513005.0,
+ "reward": 0.43836766481399536,
+ "reward_std": 0.26744046807289124,
+ "rewards/true_env_reward_fn/mean": 0.438367635011673,
+ "rewards/true_env_reward_fn/std": 0.26744046807289124,
"step": 224,
- "step_time": 4.954825423001239
+ "step_time": 10.563708176000091
},
{
"clip_ratio/high_max": 0.0,
@@ -6288,26 +6288,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 52.375,
- "completions/mean_terminated_length": 52.375,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1394256949424744,
- "epoch": 1.8292682926829267,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1480269879102707,
- "kl": 3.2915593692450784e-05,
- "learning_rate": 3.210360097450052e-07,
- "loss": 0.11319300532341003,
- "num_tokens": 923840.0,
- "reward": 0.4335233271121979,
- "reward_std": 0.31114333868026733,
- "rewards/true_env_reward_fn/mean": 0.4335233271121979,
- "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3357127904891968,
+ "epoch": 5.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07269197702407837,
+ "kl": 3.727909597728285e-05,
+ "learning_rate": 5.063851773579869e-07,
+ "loss": -0.01076439768075943,
+ "num_tokens": 5530655.0,
+ "reward": 0.6185358762741089,
+ "reward_std": 0.19721543788909912,
+ "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+ "rewards/true_env_reward_fn/std": 0.19721543788909912,
"step": 225,
- "step_time": 3.777259659000265
+ "step_time": 10.372150705000422
},
{
"clip_ratio/high_max": 0.0,
@@ -6316,26 +6316,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 64.25,
- "completions/mean_terminated_length": 64.25,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.354669451713562,
- "epoch": 1.8373983739837398,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13672730326652527,
- "kl": 2.2323275516100693e-05,
- "learning_rate": 3.173290349260188e-07,
- "loss": -0.0521523654460907,
- "num_tokens": 927882.0,
- "reward": 0.57341468334198,
- "reward_std": 0.11576741933822632,
- "rewards/true_env_reward_fn/mean": 0.57341468334198,
- "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 64.35417175292969,
+ "completions/mean_terminated_length": 64.35417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2975924611091614,
+ "epoch": 5.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0875314474105835,
+ "kl": 9.186910938296933e-05,
+ "learning_rate": 5.021284438789693e-07,
+ "loss": 0.048179637640714645,
+ "num_tokens": 5554032.0,
+ "reward": 0.45552024245262146,
+ "reward_std": 0.31553781032562256,
+ "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+ "rewards/true_env_reward_fn/std": 0.31553778052330017,
"step": 226,
- "step_time": 3.613498073998926
+ "step_time": 12.18860100899974
},
{
"clip_ratio/high_max": 0.0,
@@ -6344,26 +6344,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 64.875,
- "completions/mean_terminated_length": 64.875,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1750767230987549,
- "epoch": 1.845528455284553,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.230674147605896,
- "kl": 1.800864629331045e-05,
- "learning_rate": 3.136294625205945e-07,
- "loss": 0.058730173856019974,
- "num_tokens": 930225.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 63.208335876464844,
+ "completions/mean_terminated_length": 63.208335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2268341779708862,
+ "epoch": 5.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08318481594324112,
+ "kl": 8.052505017985823e-05,
+ "learning_rate": 4.978715561210307e-07,
+ "loss": 0.027963606640696526,
+ "num_tokens": 5582898.0,
+ "reward": 0.36310288310050964,
+ "reward_std": 0.3131144344806671,
+ "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+ "rewards/true_env_reward_fn/std": 0.3131144344806671,
"step": 227,
- "step_time": 3.3819083769976714
+ "step_time": 12.510411257000214
},
{
"clip_ratio/high_max": 0.0,
@@ -6372,26 +6372,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 40.875,
- "completions/mean_terminated_length": 40.875,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.2825847864151,
- "epoch": 1.8536585365853657,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.30391862988471985,
- "kl": 2.0106223928451072e-05,
- "learning_rate": 3.0993762379092235e-07,
- "loss": -0.10232458263635635,
- "num_tokens": 933048.0,
- "reward": 0.45667415857315063,
- "reward_std": 0.26290765404701233,
- "rewards/true_env_reward_fn/mean": 0.45667415857315063,
- "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 64.9375,
+ "completions/mean_terminated_length": 64.9375,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1668500006198883,
+ "epoch": 5.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0727877989411354,
+ "kl": 7.824771091691218e-05,
+ "learning_rate": 4.936148226420132e-07,
+ "loss": -0.01629078947007656,
+ "num_tokens": 5604791.0,
+ "reward": 0.5653349161148071,
+ "reward_std": 0.2849632799625397,
+ "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+ "rewards/true_env_reward_fn/std": 0.2849632799625397,
"step": 228,
- "step_time": 2.698590726000475
+ "step_time": 10.083805716999905
},
{
"clip_ratio/high_max": 0.0,
@@ -6400,26 +6400,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 41.125,
- "completions/mean_terminated_length": 41.125,
- "completions/min_length": 30.0,
- "completions/min_terminated_length": 30.0,
- "entropy": 1.3104369640350342,
- "epoch": 1.8617886178861789,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.28224605321884155,
- "kl": 6.227439189387951e-05,
- "learning_rate": 3.06253849306714e-07,
- "loss": -0.008679002523422241,
- "num_tokens": 938217.0,
- "reward": 0.2567799985408783,
- "reward_std": 0.2555168867111206,
- "rewards/true_env_reward_fn/mean": 0.2567799985408783,
- "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 68.29167175292969,
+ "completions/mean_terminated_length": 68.29167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.307439923286438,
+ "epoch": 5.585365853658536,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06797961890697479,
+ "kl": 6.601300174224889e-05,
+ "learning_rate": 4.893585519885763e-07,
+ "loss": -0.036383360624313354,
+ "num_tokens": 5631629.0,
+ "reward": 0.49251794815063477,
+ "reward_std": 0.22127023339271545,
+ "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+ "rewards/true_env_reward_fn/std": 0.22127023339271545,
"step": 229,
- "step_time": 2.7451177700022527
+ "step_time": 13.650024606999523
},
{
"clip_ratio/high_max": 0.0,
@@ -6428,26 +6428,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0164751410484314,
- "epoch": 1.8699186991869918,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011273886048002169,
- "kl": 1.1985231139988173e-05,
- "learning_rate": 3.0257846891560323e-07,
- "loss": 5.987301960885816e-07,
- "num_tokens": 941049.0,
- "reward": 0.6971603631973267,
- "reward_std": 0.2594861686229706,
- "rewards/true_env_reward_fn/mean": 0.6971603631973267,
- "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3524385392665863,
+ "epoch": 5.609756097560975,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.12484809756278992,
+ "kl": 0.0002558713749749586,
+ "learning_rate": 4.85103052673832e-07,
+ "loss": -0.0045075975358486176,
+ "num_tokens": 5666712.0,
+ "reward": 0.2605232000350952,
+ "reward_std": 0.22926004230976105,
+ "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+ "rewards/true_env_reward_fn/std": 0.22926005721092224,
"step": 230,
- "step_time": 2.917641182999432
+ "step_time": 15.322059910999542
},
{
"clip_ratio/high_max": 0.0,
@@ -6456,26 +6456,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 13.0,
- "completions/min_terminated_length": 13.0,
- "entropy": 1.1991845965385437,
- "epoch": 1.8780487804878048,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015363915008492768,
- "kl": 1.4284144981502322e-05,
- "learning_rate": 2.989118117136107e-07,
- "loss": 6.798551339670666e-07,
- "num_tokens": 945403.0,
- "reward": 0.31745320558547974,
- "reward_std": 0.42489534616470337,
- "rewards/true_env_reward_fn/mean": 0.31745320558547974,
- "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2202666401863098,
+ "epoch": 5.634146341463414,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07976125925779343,
+ "kl": 8.684267686476232e-05,
+ "learning_rate": 4.808486331549823e-07,
+ "loss": 0.012383833527565002,
+ "num_tokens": 5702004.0,
+ "reward": 0.20571085810661316,
+ "reward_std": 0.2265808880329132,
+ "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+ "rewards/true_env_reward_fn/std": 0.2265808880329132,
"step": 231,
- "step_time": 5.5585464220002905
+ "step_time": 14.180213787999492
},
{
"clip_ratio/high_max": 0.0,
@@ -6484,26 +6484,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 71.0,
- "completions/max_terminated_length": 71.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2020843029022217,
- "epoch": 1.886178861788618,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13684362173080444,
- "kl": 3.651866154541494e-05,
- "learning_rate": 2.952542060156773e-07,
- "loss": 0.025869816541671753,
- "num_tokens": 949126.0,
- "reward": 0.3984018564224243,
- "reward_std": 0.4202974736690521,
- "rewards/true_env_reward_fn/mean": 0.3984018564224243,
- "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "completions/max_length": 312.0,
+ "completions/max_terminated_length": 312.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3096380531787872,
+ "epoch": 5.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08384906500577927,
+ "kl": 8.733692084206268e-05,
+ "learning_rate": 4.7659560181096067e-07,
+ "loss": 0.02098608762025833,
+ "num_tokens": 5729780.0,
+ "reward": 0.4599723219871521,
+ "reward_std": 0.21680118143558502,
+ "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+ "rewards/true_env_reward_fn/std": 0.21680118143558502,
"step": 232,
- "step_time": 3.4383463869999105
+ "step_time": 21.68401394800003
},
{
"clip_ratio/high_max": 0.0,
@@ -6512,26 +6512,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 58.625,
- "completions/mean_terminated_length": 58.625,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2224581837654114,
- "epoch": 1.8943089430894309,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20249564945697784,
- "kl": 3.106597978330683e-05,
- "learning_rate": 2.9160597932626605e-07,
- "loss": -0.003915777429938316,
- "num_tokens": 952815.0,
- "reward": 0.4141089916229248,
- "reward_std": 0.3064958453178406,
- "rewards/true_env_reward_fn/mean": 0.4141089916229248,
- "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 53.520835876464844,
+ "completions/mean_terminated_length": 53.520835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2158988416194916,
+ "epoch": 5.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08225176483392715,
+ "kl": 9.274652438762132e-05,
+ "learning_rate": 4.7234426692007977e-07,
+ "loss": -0.034079790115356445,
+ "num_tokens": 5753397.0,
+ "reward": 0.5273070335388184,
+ "reward_std": 0.2194610834121704,
+ "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+ "rewards/true_env_reward_fn/std": 0.21946106851100922,
"step": 233,
- "step_time": 3.6410487339999236
+ "step_time": 7.915307780000603
},
{
"clip_ratio/high_max": 0.0,
@@ -6540,26 +6540,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1212781071662903,
- "epoch": 1.9024390243902438,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14502985775470734,
- "kl": 3.0683338081871625e-05,
- "learning_rate": 2.879674583100372e-07,
- "loss": -0.0860406681895256,
- "num_tokens": 958656.0,
- "reward": 0.30326664447784424,
- "reward_std": 0.24800051748752594,
- "rewards/true_env_reward_fn/mean": 0.30326664447784424,
- "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 78.0,
+ "completions/mean_terminated_length": 78.0,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2362721860408783,
+ "epoch": 5.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06290840357542038,
+ "kl": 7.484563047910342e-05,
+ "learning_rate": 4.6809493663768575e-07,
+ "loss": -0.008873865008354187,
+ "num_tokens": 5778869.0,
+ "reward": 0.5283026695251465,
+ "reward_std": 0.195759579539299,
+ "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+ "rewards/true_env_reward_fn/std": 0.195759579539299,
"step": 234,
- "step_time": 3.8798253620007017
+ "step_time": 10.278297286999987
},
{
"clip_ratio/high_max": 0.0,
@@ -6568,26 +6568,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 60.75,
- "completions/mean_terminated_length": 60.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.425286054611206,
- "epoch": 1.910569105691057,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2035023272037506,
- "kl": 5.512987627298571e-05,
- "learning_rate": 2.843389687625986e-07,
- "loss": -0.010439477860927582,
- "num_tokens": 962638.0,
- "reward": 0.3157375156879425,
- "reward_std": 0.46047845482826233,
- "rewards/true_env_reward_fn/mean": 0.3157375156879425,
- "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 64.47917175292969,
+ "completions/mean_terminated_length": 64.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3364675641059875,
+ "epoch": 5.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09685558825731277,
+ "kl": 0.00013002969353692606,
+ "learning_rate": 4.638479189738224e-07,
+ "loss": 0.05070740357041359,
+ "num_tokens": 5804436.0,
+ "reward": 0.44921523332595825,
+ "reward_std": 0.22693434357643127,
+ "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+ "rewards/true_env_reward_fn/std": 0.22693434357643127,
"step": 235,
- "step_time": 4.114513064998391
+ "step_time": 10.754199091999908
},
{
"clip_ratio/high_max": 0.0,
@@ -6596,26 +6596,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 120.0,
- "completions/max_terminated_length": 120.0,
- "completions/mean_length": 77.625,
- "completions/mean_terminated_length": 77.625,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4385854601860046,
- "epoch": 1.91869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12982772290706635,
- "kl": 1.2591926861205138e-05,
- "learning_rate": 2.807208355813339e-07,
- "loss": 0.09093751758337021,
- "num_tokens": 965755.0,
- "reward": 0.6140732765197754,
- "reward_std": 0.27462607622146606,
- "rewards/true_env_reward_fn/mean": 0.6140732765197754,
- "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2267529368400574,
+ "epoch": 5.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10451620817184448,
+ "kl": 6.754635069228243e-05,
+ "learning_rate": 4.596035217709039e-07,
+ "loss": -0.02925686165690422,
+ "num_tokens": 5826895.0,
+ "reward": 0.49523353576660156,
+ "reward_std": 0.15093794465065002,
+ "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+ "rewards/true_env_reward_fn/std": 0.15093792974948883,
"step": 236,
- "step_time": 5.147667763001664
+ "step_time": 9.666070583999954
},
{
"clip_ratio/high_max": 0.0,
@@ -6624,26 +6624,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 71.375,
- "completions/mean_terminated_length": 71.375,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.3476852178573608,
- "epoch": 1.9268292682926829,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14480431377887726,
- "kl": 3.8014684832887724e-05,
- "learning_rate": 2.771133827363101e-07,
- "loss": -0.07322391867637634,
- "num_tokens": 970918.0,
- "reward": 0.3427826166152954,
- "reward_std": 0.42430612444877625,
- "rewards/true_env_reward_fn/mean": 0.3427826166152954,
- "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 67.79167175292969,
+ "completions/mean_terminated_length": 67.79167175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2110519707202911,
+ "epoch": 5.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07890205830335617,
+ "kl": 0.0001245876064785989,
+ "learning_rate": 4.5536205268140286e-07,
+ "loss": -0.03693925589323044,
+ "num_tokens": 5859973.0,
+ "reward": 0.2979053258895874,
+ "reward_std": 0.23015481233596802,
+ "rewards/true_env_reward_fn/mean": 0.297905296087265,
+ "rewards/true_env_reward_fn/std": 0.2301548272371292,
"step": 237,
- "step_time": 4.67846887900123
+ "step_time": 13.971699990999241
},
{
"clip_ratio/high_max": 0.0,
@@ -6652,26 +6652,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 221.0,
- "completions/max_terminated_length": 221.0,
- "completions/mean_length": 94.25,
- "completions/mean_terminated_length": 94.25,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.3279914855957031,
- "epoch": 1.934959349593496,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10826019197702408,
- "kl": 1.3074863090878353e-05,
- "learning_rate": 2.7351693324127037e-07,
- "loss": -0.05556309223175049,
- "num_tokens": 975120.0,
- "reward": 0.64573073387146,
- "reward_std": 0.22739914059638977,
- "rewards/true_env_reward_fn/mean": 0.64573073387146,
- "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 65.52083587646484,
+ "completions/mean_terminated_length": 65.52083587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2291057407855988,
+ "epoch": 5.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09152546525001526,
+ "kl": 0.00013101351214572787,
+ "learning_rate": 4.511238191455491e-07,
+ "loss": 0.043641336262226105,
+ "num_tokens": 5886118.0,
+ "reward": 0.41334670782089233,
+ "reward_std": 0.19480590522289276,
+ "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+ "rewards/true_env_reward_fn/std": 0.19480590522289276,
"step": 238,
- "step_time": 9.220254810001279
+ "step_time": 10.805698846999803
},
{
"clip_ratio/high_max": 0.0,
@@ -6680,26 +6680,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3695034384727478,
- "epoch": 1.943089430894309,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12166028469800949,
- "kl": 2.6563114261080045e-05,
- "learning_rate": 2.6993180912471055e-07,
- "loss": -0.05334407091140747,
- "num_tokens": 980254.0,
- "reward": 0.37203267216682434,
- "reward_std": 0.20089927315711975,
- "rewards/true_env_reward_fn/mean": 0.37203267216682434,
- "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 58.833335876464844,
+ "completions/mean_terminated_length": 58.833335876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3134913444519043,
+ "epoch": 5.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07343924045562744,
+ "kl": 0.00019655993492051493,
+ "learning_rate": 4.4688912836904533e-07,
+ "loss": 0.015309082344174385,
+ "num_tokens": 5908334.0,
+ "reward": 0.4438478946685791,
+ "reward_std": 0.27188754081726074,
+ "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+ "rewards/true_env_reward_fn/std": 0.27188754081726074,
"step": 239,
- "step_time": 4.224964968001586
+ "step_time": 9.688736522
},
{
"clip_ratio/high_max": 0.0,
@@ -6708,26 +6708,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.75,
- "completions/mean_terminated_length": 53.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.247464120388031,
- "epoch": 1.951219512195122,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2346416860818863,
- "kl": 5.1520751185307745e-05,
- "learning_rate": 2.6635833140104405e-07,
- "loss": -0.0905834436416626,
- "num_tokens": 983244.0,
- "reward": 0.575507640838623,
- "reward_std": 0.17391785979270935,
- "rewards/true_env_reward_fn/mean": 0.575507640838623,
- "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "completions/max_length": 464.0,
+ "completions/max_terminated_length": 464.0,
+ "completions/mean_length": 79.14583587646484,
+ "completions/mean_terminated_length": 79.14583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3148745000362396,
+ "epoch": 5.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06567981839179993,
+ "kl": 4.445325248525478e-05,
+ "learning_rate": 4.4265828730079977e-07,
+ "loss": 0.021218176931142807,
+ "num_tokens": 5929645.0,
+ "reward": 0.5866342186927795,
+ "reward_std": 0.13780196011066437,
+ "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+ "rewards/true_env_reward_fn/std": 0.13780196011066437,
"step": 240,
- "step_time": 3.357481237999309
+ "step_time": 27.046819901000163
},
{
"clip_ratio/high_max": 0.0,
@@ -6736,26 +6736,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2175387144088745,
- "epoch": 1.959349593495935,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1870066374540329,
- "kl": 2.2185965462995227e-05,
- "learning_rate": 2.6279682004185894e-07,
- "loss": -0.07365687191486359,
- "num_tokens": 985574.0,
- "reward": 0.6635305881500244,
- "reward_std": 0.1901332288980484,
- "rewards/true_env_reward_fn/mean": 0.6635305881500244,
- "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "completions/max_length": 364.0,
+ "completions/max_terminated_length": 364.0,
+ "completions/mean_length": 70.33333587646484,
+ "completions/mean_terminated_length": 70.33333587646484,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2104995846748352,
+ "epoch": 5.878048780487805,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.042626503854990005,
+ "kl": 6.413207393052289e-05,
+ "learning_rate": 4.3843160261067653e-07,
+ "loss": -0.0065308245830237865,
+ "num_tokens": 5950005.0,
+ "reward": 0.5593677759170532,
+ "reward_std": 0.23993276059627533,
+ "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+ "rewards/true_env_reward_fn/std": 0.23993274569511414,
"step": 241,
- "step_time": 4.527591582998866
+ "step_time": 22.238758486000734
},
{
"clip_ratio/high_max": 0.0,
@@ -6764,26 +6764,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.25,
- "completions/mean_terminated_length": 76.25,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2727615237236023,
- "epoch": 1.967479674796748,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.226071804529056e-05,
- "kl": 1.4014385214977665e-05,
- "learning_rate": 2.592475939472668e-07,
- "loss": 7.015369192231447e-07,
- "num_tokens": 988868.0,
- "reward": 0.7316612601280212,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.7316612601280212,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2132116258144379,
+ "epoch": 5.902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05718924477696419,
+ "kl": 6.140609366411809e-05,
+ "learning_rate": 4.342093806672678e-07,
+ "loss": 0.012734346091747284,
+ "num_tokens": 5974626.0,
+ "reward": 0.4349059462547302,
+ "reward_std": 0.24915602803230286,
+ "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+ "rewards/true_env_reward_fn/std": 0.24915601313114166,
"step": 242,
- "step_time": 4.238274277000528
+ "step_time": 9.239750460000323
},
{
"clip_ratio/high_max": 0.0,
@@ -6791,27 +6791,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 57.5,
- "completions/mean_terminated_length": 57.5,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1519948840141296,
- "epoch": 1.975609756097561,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12447996437549591,
- "kl": 4.700180943473242e-05,
- "learning_rate": 2.557109709173482e-07,
- "loss": 0.08819369971752167,
- "num_tokens": 994068.0,
- "reward": 0.3696666657924652,
- "reward_std": 0.18488828837871552,
- "rewards/true_env_reward_fn/mean": 0.3696666657924652,
- "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 67.04167175292969,
+ "completions/mean_terminated_length": 57.574466705322266,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.7212344706058502,
+ "epoch": 5.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08872146159410477,
+ "kl": 0.00011613740025495645,
+ "learning_rate": 4.2999192751568557e-07,
+ "loss": -0.021363887935876846,
+ "num_tokens": 5997300.0,
+ "reward": 0.4436037540435791,
+ "reward_std": 0.28323379158973694,
+ "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+ "rewards/true_env_reward_fn/std": 0.28323376178741455,
"step": 243,
- "step_time": 3.3367313500002638
+ "step_time": 29.767976787999487
},
{
"clip_ratio/high_max": 0.0,
@@ -6820,26 +6820,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 115.0,
- "completions/max_terminated_length": 115.0,
- "completions/mean_length": 68.25,
- "completions/mean_terminated_length": 68.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0537148416042328,
- "epoch": 1.9837398373983741,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.17681685090065002,
- "kl": 3.6240851841284893e-05,
- "learning_rate": 2.521872676236972e-07,
- "loss": 0.05281040072441101,
- "num_tokens": 1000650.0,
- "reward": 0.12209999561309814,
- "reward_std": 0.2502918243408203,
- "rewards/true_env_reward_fn/mean": 0.12209999561309814,
- "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.083335876464844,
+ "completions/mean_terminated_length": 61.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2353634238243103,
+ "epoch": 5.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07658001780509949,
+ "kl": 9.086773115996039e-05,
+ "learning_rate": 4.257795488553798e-07,
+ "loss": -0.005057391710579395,
+ "num_tokens": 6021752.0,
+ "reward": 0.49971556663513184,
+ "reward_std": 0.2643933594226837,
+ "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+ "rewards/true_env_reward_fn/std": 0.2643933594226837,
"step": 244,
- "step_time": 5.2957401019993995
+ "step_time": 9.764708648999658
},
{
"clip_ratio/high_max": 0.0,
@@ -6847,27 +6847,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 140.0,
- "completions/max_terminated_length": 140.0,
- "completions/mean_length": 95.375,
- "completions/mean_terminated_length": 95.375,
- "completions/min_length": 61.0,
- "completions/min_terminated_length": 61.0,
- "entropy": 1.3126497864723206,
- "epoch": 1.9918699186991868,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16036106646060944,
- "kl": 6.061139720259234e-05,
- "learning_rate": 2.48676799581066e-07,
- "loss": 0.09418506920337677,
- "num_tokens": 1005737.0,
- "reward": 0.31175702810287476,
- "reward_std": 0.38867074251174927,
- "rewards/true_env_reward_fn/mean": 0.31175702810287476,
- "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 86.72917175292969,
+ "completions/mean_terminated_length": 77.68084716796875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2082330882549286,
+ "epoch": 5.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.059102799743413925,
+ "kl": 5.0710960749711376e-05,
+ "learning_rate": 4.215725500179787e-07,
+ "loss": 0.17436102032661438,
+ "num_tokens": 6049747.0,
+ "reward": 0.46306928992271423,
+ "reward_std": 0.26072925329208374,
+ "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+ "rewards/true_env_reward_fn/std": 0.26072925329208374,
"step": 245,
- "step_time": 6.259088058999623
+ "step_time": 31.997988874000384
},
{
"clip_ratio/high_max": 0.0,
@@ -6876,26 +6876,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 48.5,
- "completions/mean_terminated_length": 48.5,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.21333646774292,
- "epoch": 2.0,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23997871577739716,
- "kl": 3.1378609492094256e-05,
- "learning_rate": 2.4517988111911313e-07,
- "loss": 0.010592922568321228,
- "num_tokens": 1010869.0,
- "reward": 0.33381664752960205,
- "reward_std": 0.18213039636611938,
- "rewards/true_env_reward_fn/mean": 0.33381664752960205,
- "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 61.35416793823242,
+ "completions/mean_terminated_length": 61.35416793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2454268038272858,
+ "epoch": 6.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07332625985145569,
+ "kl": 0.0001107546740968246,
+ "learning_rate": 4.1737123594515755e-07,
+ "loss": -0.013426866382360458,
+ "num_tokens": 6072668.0,
+ "reward": 0.45415109395980835,
+ "reward_std": 0.2937946617603302,
+ "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+ "rewards/true_env_reward_fn/std": 0.2937946617603302,
"step": 246,
- "step_time": 3.111915630997828
+ "step_time": 9.587768273000165
},
{
"clip_ratio/high_max": 0.0,
@@ -6904,26 +6904,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 58.0,
- "completions/min_terminated_length": 58.0,
- "entropy": 1.3962982892990112,
- "epoch": 2.008130081300813,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21355785429477692,
- "kl": 3.789625407080166e-05,
- "learning_rate": 2.4169682535425927e-07,
- "loss": 0.025682777166366577,
- "num_tokens": 1014876.0,
- "reward": 0.35749268531799316,
- "reward_std": 0.29738906025886536,
- "rewards/true_env_reward_fn/mean": 0.35749268531799316,
- "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 71.5625,
+ "completions/mean_terminated_length": 71.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1738699078559875,
+ "epoch": 6.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06093747541308403,
+ "kl": 7.893411338955048e-05,
+ "learning_rate": 4.131759111665348e-07,
+ "loss": 0.022837676107883453,
+ "num_tokens": 6095263.0,
+ "reward": 0.5188159346580505,
+ "reward_std": 0.3265886902809143,
+ "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+ "rewards/true_env_reward_fn/std": 0.3265886902809143,
"step": 247,
- "step_time": 3.35338095500083
+ "step_time": 14.232978527999876
},
{
"clip_ratio/high_max": 0.0,
@@ -6932,26 +6932,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 73.875,
- "completions/mean_terminated_length": 73.875,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.463137686252594,
- "epoch": 2.016260162601626,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18188992142677307,
- "kl": 1.746804719005013e-05,
- "learning_rate": 2.382279441616492e-07,
- "loss": -0.17857304215431213,
- "num_tokens": 1018383.0,
- "reward": 0.5329012274742126,
- "reward_std": 0.055823445320129395,
- "rewards/true_env_reward_fn/mean": 0.5329012274742126,
- "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3300949931144714,
+ "epoch": 6.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08098509907722473,
+ "kl": 0.00010828049380506855,
+ "learning_rate": 4.0898687977759887e-07,
+ "loss": 0.01631344109773636,
+ "num_tokens": 6125380.0,
+ "reward": 0.3338983952999115,
+ "reward_std": 0.19050820171833038,
+ "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+ "rewards/true_env_reward_fn/std": 0.19050820171833038,
"step": 248,
- "step_time": 5.210386754000865
+ "step_time": 13.248441182000079
},
{
"clip_ratio/high_max": 0.0,
@@ -6960,26 +6960,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 211.0,
- "completions/max_terminated_length": 211.0,
- "completions/mean_length": 79.25,
- "completions/mean_terminated_length": 79.25,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.4478936195373535,
- "epoch": 2.024390243902439,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.744662434561178e-05,
- "kl": 1.3336490155779757e-05,
- "learning_rate": 2.3477354814722762e-07,
- "loss": 6.725406365148956e-07,
- "num_tokens": 1022753.0,
- "reward": 0.5905972719192505,
- "reward_std": 0.15080371499061584,
- "rewards/true_env_reward_fn/mean": 0.5905972719192505,
- "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "completions/max_length": 396.0,
+ "completions/max_terminated_length": 396.0,
+ "completions/mean_length": 88.14583587646484,
+ "completions/mean_terminated_length": 88.14583587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3161276876926422,
+ "epoch": 6.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07784765958786011,
+ "kl": 4.819030800717883e-05,
+ "learning_rate": 4.0480444541766575e-07,
+ "loss": 0.12299837917089462,
+ "num_tokens": 6151955.0,
+ "reward": 0.5446512699127197,
+ "reward_std": 0.2611033618450165,
+ "rewards/true_env_reward_fn/mean": 0.544651210308075,
+ "rewards/true_env_reward_fn/std": 0.2611033618450165,
"step": 249,
- "step_time": 8.798317029002646
+ "step_time": 25.232192139999825
},
{
"clip_ratio/high_max": 0.0,
@@ -6988,26 +6988,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 64.75,
- "completions/mean_terminated_length": 64.75,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.37166029214859,
- "epoch": 2.032520325203252,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1542455554008484,
- "kl": 2.0379054603836266e-05,
- "learning_rate": 2.313339466199264e-07,
- "loss": -0.037539318203926086,
- "num_tokens": 1025971.0,
- "reward": 0.6065863966941833,
- "reward_std": 0.032470256090164185,
- "rewards/true_env_reward_fn/mean": 0.6065863966941833,
- "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 69.97917175292969,
+ "completions/mean_terminated_length": 69.97917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2318329215049744,
+ "epoch": 6.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06498592346906662,
+ "kl": 3.854301621686318e-05,
+ "learning_rate": 4.0062891124787e-07,
+ "loss": -0.04302535578608513,
+ "num_tokens": 6174898.0,
+ "reward": 0.6081289052963257,
+ "reward_std": 0.24437586963176727,
+ "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+ "rewards/true_env_reward_fn/std": 0.24437588453292847,
"step": 250,
- "step_time": 4.096263454999644
+ "step_time": 10.385816780999448
},
{
"clip_ratio/high_max": 0.0,
@@ -7016,26 +7016,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 60.5,
- "completions/mean_terminated_length": 60.5,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.4042693972587585,
- "epoch": 2.040650406504065,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.163822203874588,
- "kl": 3.525477495713858e-05,
- "learning_rate": 2.2790944756396916e-07,
- "loss": 0.03408379852771759,
- "num_tokens": 1029415.0,
- "reward": 0.37829869985580444,
- "reward_std": 0.2773255407810211,
- "rewards/true_env_reward_fn/mean": 0.37829869985580444,
- "rewards/true_env_reward_fn/std": 0.2773255407810211,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.91666793823242,
+ "completions/mean_terminated_length": 60.91666793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2425517737865448,
+ "epoch": 6.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08402577042579651,
+ "kl": 0.00010545800250838511,
+ "learning_rate": 3.9646057992918966e-07,
+ "loss": 0.006328321993350983,
+ "num_tokens": 6203582.0,
+ "reward": 0.30250340700149536,
+ "reward_std": 0.3084425926208496,
+ "rewards/true_env_reward_fn/mean": 0.30250340700149536,
+ "rewards/true_env_reward_fn/std": 0.3084425628185272,
"step": 251,
- "step_time": 3.9464334140011488
+ "step_time": 9.779451584999606
},
{
"clip_ratio/high_max": 0.0,
@@ -7044,26 +7044,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 181.0,
- "completions/max_terminated_length": 181.0,
- "completions/mean_length": 73.75,
- "completions/mean_terminated_length": 73.75,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.2268111109733582,
- "epoch": 2.048780487804878,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1961166262626648,
- "kl": 3.528672823449597e-05,
- "learning_rate": 2.2450035761129391e-07,
- "loss": 0.2999379336833954,
- "num_tokens": 1032213.0,
- "reward": 0.6772161722183228,
- "reward_std": 0.31218820810317993,
- "rewards/true_env_reward_fn/mean": 0.6772161722183228,
- "rewards/true_env_reward_fn/std": 0.31218820810317993,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.5,
+ "completions/mean_terminated_length": 65.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2977190017700195,
+ "epoch": 6.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372989296913147,
+ "kl": 0.00017708011364447884,
+ "learning_rate": 3.9229975360050934e-07,
+ "loss": 0.04882372170686722,
+ "num_tokens": 6227486.0,
+ "reward": 0.5265982151031494,
+ "reward_std": 0.31736499071121216,
+ "rewards/true_env_reward_fn/mean": 0.5265981554985046,
+ "rewards/true_env_reward_fn/std": 0.31736496090888977,
"step": 252,
- "step_time": 7.437029113001699
+ "step_time": 10.318136508000407
},
{
"clip_ratio/high_max": 0.0,
@@ -7072,26 +7072,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.360656499862671,
- "epoch": 2.0569105691056913,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.24334783852100372,
- "kl": 5.1042834456893615e-05,
- "learning_rate": 2.2110698201409787e-07,
- "loss": 0.05509951710700989,
- "num_tokens": 1038598.0,
- "reward": 0.2947666645050049,
- "reward_std": 0.19891902804374695,
- "rewards/true_env_reward_fn/mean": 0.2947666645050049,
- "rewards/true_env_reward_fn/std": 0.19891902804374695,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 71.125,
+ "completions/mean_terminated_length": 71.125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3135383129119873,
+ "epoch": 6.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05451377481222153,
+ "kl": 6.562464477610774e-05,
+ "learning_rate": 3.8814673385671893e-07,
+ "loss": 0.01392640545964241,
+ "num_tokens": 6255436.0,
+ "reward": 0.4130678176879883,
+ "reward_std": 0.1573377549648285,
+ "rewards/true_env_reward_fn/mean": 0.4130678176879883,
+ "rewards/true_env_reward_fn/std": 0.15733776986598969,
"step": 253,
- "step_time": 3.7359043900014512
+ "step_time": 9.396596211999622
},
{
"clip_ratio/high_max": 0.0,
@@ -7100,26 +7100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 74.75,
- "completions/mean_terminated_length": 74.75,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4314632415771484,
- "epoch": 2.065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1960419863462448,
- "kl": 6.17889963905327e-05,
- "learning_rate": 2.1772962461750342e-07,
- "loss": 0.0954262986779213,
- "num_tokens": 1043268.0,
- "reward": 0.4036714732646942,
- "reward_std": 0.42137831449508667,
- "rewards/true_env_reward_fn/mean": 0.4036714732646942,
- "rewards/true_env_reward_fn/std": 0.42137834429740906,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 73.85417175292969,
+ "completions/mean_terminated_length": 73.85417175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3749704957008362,
+ "epoch": 6.195121951219512,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05664386227726936,
+ "kl": 6.744195252395002e-05,
+ "learning_rate": 3.840018217268526e-07,
+ "loss": -0.04205852374434471,
+ "num_tokens": 6281557.0,
+ "reward": 0.5188004970550537,
+ "reward_std": 0.2932124733924866,
+ "rewards/true_env_reward_fn/mean": 0.5188004970550537,
+ "rewards/true_env_reward_fn/std": 0.2932124435901642,
"step": 254,
- "step_time": 5.565175547999388
+ "step_time": 19.77463799499992
},
{
"clip_ratio/high_max": 0.0,
@@ -7128,26 +7128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 68.625,
- "completions/mean_terminated_length": 68.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.0521443486213684,
- "epoch": 2.073170731707317,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.464648271910846e-05,
- "kl": 1.2018902907584561e-05,
- "learning_rate": 2.1436858783235338e-07,
- "loss": 6.008343689245521e-07,
- "num_tokens": 1046517.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2866049408912659,
+ "epoch": 6.219512195121951,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06639927625656128,
+ "kl": 5.235667686065426e-05,
+ "learning_rate": 3.798653176522696e-07,
+ "loss": -0.019558893516659737,
+ "num_tokens": 6302850.0,
+ "reward": 0.5025100111961365,
+ "reward_std": 0.1717289537191391,
+ "rewards/true_env_reward_fn/mean": 0.5025100111961365,
+ "rewards/true_env_reward_fn/std": 0.1717289537191391,
"step": 255,
- "step_time": 4.379171047001364
+ "step_time": 9.634558264001043
},
{
"clip_ratio/high_max": 0.0,
@@ -7156,26 +7156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 55.25,
- "completions/mean_terminated_length": 55.25,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.091518223285675,
- "epoch": 2.08130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 9.263061656383798e-05,
- "kl": 1.4573892713087844e-05,
- "learning_rate": 2.110241726081317e-07,
- "loss": 7.304333848878741e-07,
- "num_tokens": 1050439.0,
- "reward": 0.6024306416511536,
- "reward_std": 0.13815335929393768,
- "rewards/true_env_reward_fn/mean": 0.6024306416511536,
- "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.3105897009372711,
+ "epoch": 6.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07659970223903656,
+ "kl": 8.12946273072157e-05,
+ "learning_rate": 3.7573752146487636e-07,
+ "loss": 0.05201243981719017,
+ "num_tokens": 6329269.0,
+ "reward": 0.5027985572814941,
+ "reward_std": 0.2521378695964813,
+ "rewards/true_env_reward_fn/mean": 0.5027985572814941,
+ "rewards/true_env_reward_fn/std": 0.2521378993988037,
"step": 256,
- "step_time": 3.8379976090000127
+ "step_time": 11.07390475200009
},
{
"clip_ratio/high_max": 0.0,
@@ -7184,26 +7184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2301559448242188,
- "epoch": 2.089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17180238664150238,
- "kl": 2.236898035334889e-05,
- "learning_rate": 2.076966784060165e-07,
- "loss": 0.023751959204673767,
- "num_tokens": 1054578.0,
- "reward": 0.4112047851085663,
- "reward_std": 0.05330020561814308,
- "rewards/true_env_reward_fn/mean": 0.4112047851085663,
- "rewards/true_env_reward_fn/std": 0.05330020561814308,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.3154918253421783,
+ "epoch": 6.2682926829268295,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10612925887107849,
+ "kl": 0.00010100230429088697,
+ "learning_rate": 3.7161873236539386e-07,
+ "loss": 0.0214182510972023,
+ "num_tokens": 6356233.0,
+ "reward": 0.2817384600639343,
+ "reward_std": 0.3363598883152008,
+ "rewards/true_env_reward_fn/mean": 0.2817384600639343,
+ "rewards/true_env_reward_fn/std": 0.3363598883152008,
"step": 257,
- "step_time": 4.424114469000415
+ "step_time": 10.784447634000117
},
{
"clip_ratio/high_max": 0.0,
@@ -7212,26 +7212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 52.125,
- "completions/mean_terminated_length": 52.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2734522223472595,
- "epoch": 2.097560975609756,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2274128645658493,
- "kl": 5.3426387239596806e-05,
- "learning_rate": 2.043864031720667e-07,
- "loss": 0.05165906995534897,
- "num_tokens": 1060579.0,
- "reward": 0.2129499912261963,
- "reward_std": 0.2476053088903427,
- "rewards/true_env_reward_fn/mean": 0.2129499912261963,
- "rewards/true_env_reward_fn/std": 0.2476053088903427,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 62.770835876464844,
+ "completions/mean_terminated_length": 62.770835876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1986172497272491,
+ "epoch": 6.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08129199594259262,
+ "kl": 0.00011449725025158841,
+ "learning_rate": 3.6750924890166923e-07,
+ "loss": -0.05941678211092949,
+ "num_tokens": 6375046.0,
+ "reward": 0.5760313272476196,
+ "reward_std": 0.24430812895298004,
+ "rewards/true_env_reward_fn/mean": 0.5760312676429749,
+ "rewards/true_env_reward_fn/std": 0.24430814385414124,
"step": 258,
- "step_time": 3.4830677139998443
+ "step_time": 9.467202022999572
},
{
"clip_ratio/high_max": 0.0,
@@ -7240,26 +7240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 54.375,
- "completions/mean_terminated_length": 54.375,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2013150453567505,
- "epoch": 2.105691056910569,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20796725153923035,
- "kl": 8.34841157484334e-05,
- "learning_rate": 2.0109364331054297e-07,
- "loss": 0.04251064360141754,
- "num_tokens": 1065318.0,
- "reward": 0.2038009911775589,
- "reward_std": 0.3393669128417969,
- "rewards/true_env_reward_fn/mean": 0.2038009911775589,
- "rewards/true_env_reward_fn/std": 0.3393669128417969,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2520000636577606,
+ "epoch": 6.317073170731708,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.06876237690448761,
+ "kl": 0.0001509120993432589,
+ "learning_rate": 3.6340936894703713e-07,
+ "loss": 0.015932239592075348,
+ "num_tokens": 6397298.0,
+ "reward": 0.4944794774055481,
+ "reward_std": 0.24476772546768188,
+ "rewards/true_env_reward_fn/mean": 0.4944794476032257,
+ "rewards/true_env_reward_fn/std": 0.24476774036884308,
"step": 259,
- "step_time": 3.34712773299907
+ "step_time": 10.47640546699995
},
{
"clip_ratio/high_max": 0.0,
@@ -7268,26 +7268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 52.25,
- "completions/mean_terminated_length": 52.25,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.3005307912826538,
- "epoch": 2.113821138211382,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2342396229505539,
- "kl": 2.9608699151140172e-05,
- "learning_rate": 1.9781869365736777e-07,
- "loss": -0.11542908847332001,
- "num_tokens": 1068352.0,
- "reward": 0.5755212306976318,
- "reward_std": 0.1629202961921692,
- "rewards/true_env_reward_fn/mean": 0.5755212306976318,
- "rewards/true_env_reward_fn/std": 0.1629202961921692,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3675504922866821,
+ "epoch": 6.341463414634147,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06423573940992355,
+ "kl": 0.00010545238592385431,
+ "learning_rate": 3.593193896787277e-07,
+ "loss": 0.006066613830626011,
+ "num_tokens": 6423927.0,
+ "reward": 0.4633293151855469,
+ "reward_std": 0.2953638732433319,
+ "rewards/true_env_reward_fn/mean": 0.4633293151855469,
+ "rewards/true_env_reward_fn/std": 0.2953638732433319,
"step": 260,
- "step_time": 3.0931306170004973
+ "step_time": 11.672983966999709
},
{
"clip_ratio/high_max": 0.0,
@@ -7296,26 +7296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 70.875,
- "completions/mean_terminated_length": 70.875,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1381222009658813,
- "epoch": 2.1219512195121952,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13169968128204346,
- "kl": 1.4705466128361877e-05,
- "learning_rate": 1.9456184745372558e-07,
- "loss": -0.026440951973199844,
- "num_tokens": 1070847.0,
- "reward": 0.5704532265663147,
- "reward_std": 0.13928835093975067,
- "rewards/true_env_reward_fn/mean": 0.5704532265663147,
- "rewards/true_env_reward_fn/std": 0.13928835093975067,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 74.5,
+ "completions/mean_terminated_length": 74.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3533118963241577,
+ "epoch": 6.365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07830996066331863,
+ "kl": 8.717951732251095e-05,
+ "learning_rate": 3.552396075563257e-07,
+ "loss": -0.021772567182779312,
+ "num_tokens": 6451487.0,
+ "reward": 0.3543766736984253,
+ "reward_std": 0.33776554465293884,
+ "rewards/true_env_reward_fn/mean": 0.3543766736984253,
+ "rewards/true_env_reward_fn/std": 0.33776557445526123,
"step": 261,
- "step_time": 3.8980969309996
+ "step_time": 13.734938852000141
},
{
"clip_ratio/high_max": 0.0,
@@ -7324,26 +7324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.241390347480774,
- "epoch": 2.130081300813008,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010450400441186503,
- "kl": 1.6406540453317575e-05,
- "learning_rate": 1.9132339631980622e-07,
- "loss": 8.202600838558283e-07,
- "num_tokens": 1076153.0,
- "reward": 0.41493332386016846,
- "reward_std": 0.06228968873620033,
- "rewards/true_env_reward_fn/mean": 0.41493332386016846,
- "rewards/true_env_reward_fn/std": 0.06228969246149063,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 71.45833587646484,
+ "completions/mean_terminated_length": 71.45833587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1788119971752167,
+ "epoch": 6.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0994381383061409,
+ "kl": 0.00011037426884286106,
+ "learning_rate": 3.511703183002827e-07,
+ "loss": 0.008012240752577782,
+ "num_tokens": 6476141.0,
+ "reward": 0.45457857847213745,
+ "reward_std": 0.24262367188930511,
+ "rewards/true_env_reward_fn/mean": 0.45457854866981506,
+ "rewards/true_env_reward_fn/std": 0.24262367188930511,
"step": 262,
- "step_time": 3.6601423579995753
+ "step_time": 10.59573544200066
},
{
"clip_ratio/high_max": 0.0,
@@ -7352,26 +7352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 81.0,
- "completions/mean_terminated_length": 81.0,
- "completions/min_length": 67.0,
- "completions/min_terminated_length": 67.0,
- "entropy": 1.1326860189437866,
- "epoch": 2.138211382113821,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001754833065206185,
- "kl": 1.4390577689482598e-05,
- "learning_rate": 1.881036302286923e-07,
- "loss": 7.2446778176527e-07,
- "num_tokens": 1080537.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.293170690536499,
+ "epoch": 6.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10550480335950851,
+ "kl": 0.00020241059792169835,
+ "learning_rate": 3.4711181687048106e-07,
+ "loss": 0.02680305391550064,
+ "num_tokens": 6499670.0,
+ "reward": 0.33261698484420776,
+ "reward_std": 0.31543657183647156,
+ "rewards/true_env_reward_fn/mean": 0.3326169550418854,
+ "rewards/true_env_reward_fn/std": 0.31543657183647156,
"step": 263,
- "step_time": 4.190891189999093
+ "step_time": 10.02452396199942
},
{
"clip_ratio/high_max": 0.0,
@@ -7380,26 +7380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 103.0,
- "completions/max_terminated_length": 103.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.353486955165863,
- "epoch": 2.1463414634146343,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.215454563498497,
- "kl": 6.859865607111715e-05,
- "learning_rate": 1.8490283748039515e-07,
- "loss": 0.0181141197681427,
- "num_tokens": 1084056.0,
- "reward": 0.3049938380718231,
- "reward_std": 0.4605039656162262,
- "rewards/true_env_reward_fn/mean": 0.3049938380718231,
- "rewards/true_env_reward_fn/std": 0.4605039954185486,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 89.83333587646484,
+ "completions/mean_terminated_length": 89.83333587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2982321977615356,
+ "epoch": 6.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06412192434072495,
+ "kl": 6.536830551340245e-05,
+ "learning_rate": 3.4306439744485447e-07,
+ "loss": -0.08396965265274048,
+ "num_tokens": 6522630.0,
+ "reward": 0.5518762469291687,
+ "reward_std": 0.21314994990825653,
+ "rewards/true_env_reward_fn/mean": 0.5518762469291687,
+ "rewards/true_env_reward_fn/std": 0.21314994990825653,
"step": 264,
- "step_time": 4.504906432999633
+ "step_time": 20.82168071200067
},
{
"clip_ratio/high_max": 0.0,
@@ -7408,26 +7408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 127.0,
- "completions/max_terminated_length": 127.0,
- "completions/mean_length": 70.375,
- "completions/mean_terminated_length": 70.375,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 63.85416793823242,
+ "completions/mean_terminated_length": 63.85416793823242,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.0113105773925781,
- "epoch": 2.154471544715447,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10072485357522964,
- "kl": 1.4604076568502933e-05,
- "learning_rate": 1.8172130467604085e-07,
- "loss": -0.041721273213624954,
- "num_tokens": 1090171.0,
- "reward": 0.47745320200920105,
- "reward_std": 0.2929421067237854,
- "rewards/true_env_reward_fn/mean": 0.47745320200920105,
- "rewards/true_env_reward_fn/std": 0.2929421067237854,
+ "entropy": 1.2279660403728485,
+ "epoch": 6.463414634146342,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10119301080703735,
+ "kl": 0.00010847221983567579,
+ "learning_rate": 3.3902835339806456e-07,
+ "loss": 0.01589711755514145,
+ "num_tokens": 6548183.0,
+ "reward": 0.4527897238731384,
+ "reward_std": 0.2534019947052002,
+ "rewards/true_env_reward_fn/mean": 0.45278969407081604,
+ "rewards/true_env_reward_fn/std": 0.2534019649028778,
"step": 265,
- "step_time": 6.299696521999067
+ "step_time": 8.63894235699945
},
{
"clip_ratio/high_max": 0.0,
@@ -7436,26 +7436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 57.375,
- "completions/mean_terminated_length": 57.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3366597294807434,
- "epoch": 2.16260162601626,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2297964245080948,
- "kl": 5.677436638507061e-05,
- "learning_rate": 1.785593166922062e-07,
- "loss": 0.20361776649951935,
- "num_tokens": 1094358.0,
- "reward": 0.3835672438144684,
- "reward_std": 0.4339357614517212,
- "rewards/true_env_reward_fn/mean": 0.3835672438144684,
- "rewards/true_env_reward_fn/std": 0.4339357912540436,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 60.54166793823242,
+ "completions/mean_terminated_length": 60.54166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1612891554832458,
+ "epoch": 6.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0825633853673935,
+ "kl": 7.345602170971688e-05,
+ "learning_rate": 3.3500397728023534e-07,
+ "loss": 0.057398565113544464,
+ "num_tokens": 6571537.0,
+ "reward": 0.5196421146392822,
+ "reward_std": 0.18547315895557404,
+ "rewards/true_env_reward_fn/mean": 0.5196421146392822,
+ "rewards/true_env_reward_fn/std": 0.18547315895557404,
"step": 266,
- "step_time": 4.192992550000781
+ "step_time": 10.01155260700034
},
{
"clip_ratio/high_max": 0.0,
@@ -7464,26 +7464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 65.375,
- "completions/mean_terminated_length": 65.375,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.4040917754173279,
- "epoch": 2.1707317073170733,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2408597469329834,
+ "epoch": 6.512195121951219,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.1913405954837799,
- "kl": 2.2514723241329193e-05,
- "learning_rate": 1.7541715665541276e-07,
- "loss": -0.12278837710618973,
- "num_tokens": 1099685.0,
- "reward": 0.5316476821899414,
- "reward_std": 0.1867343932390213,
- "rewards/true_env_reward_fn/mean": 0.5316476821899414,
- "rewards/true_env_reward_fn/std": 0.1867344230413437,
+ "grad_norm": 0.09217255562543869,
+ "kl": 9.46905902310391e-05,
+ "learning_rate": 3.3099156079574867e-07,
+ "loss": 0.026750221848487854,
+ "num_tokens": 6596010.0,
+ "reward": 0.4330406188964844,
+ "reward_std": 0.20423907041549683,
+ "rewards/true_env_reward_fn/mean": 0.4330406188964844,
+ "rewards/true_env_reward_fn/std": 0.20423908531665802,
"step": 267,
- "step_time": 4.577502725998784
+ "step_time": 20.002466699999786
},
{
"clip_ratio/high_max": 0.0,
@@ -7492,26 +7492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2707499265670776,
- "epoch": 2.178861788617886,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1252160668373108,
- "kl": 2.1676510186807718e-05,
- "learning_rate": 1.722951059167737e-07,
- "loss": 0.030697450041770935,
- "num_tokens": 1103244.0,
- "reward": 0.47949954867362976,
- "reward_std": 0.10323704034090042,
- "rewards/true_env_reward_fn/mean": 0.47949954867362976,
- "rewards/true_env_reward_fn/std": 0.10323705524206161,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 64.83333587646484,
+ "completions/mean_terminated_length": 64.83333587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2907497882843018,
+ "epoch": 6.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07903403043746948,
+ "kl": 0.00015101409735507332,
+ "learning_rate": 3.269913947820998e-07,
+ "loss": 0.02006390690803528,
+ "num_tokens": 6622106.0,
+ "reward": 0.4544076919555664,
+ "reward_std": 0.26717478036880493,
+ "rewards/true_env_reward_fn/mean": 0.4544076919555664,
+ "rewards/true_env_reward_fn/std": 0.2671748101711273,
"step": 268,
- "step_time": 4.3898782989999745
+ "step_time": 8.818348709999555
},
{
"clip_ratio/high_max": 0.0,
@@ -7520,26 +7520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3778526186943054,
- "epoch": 2.186991869918699,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19924476742744446,
- "kl": 5.189802323002368e-05,
- "learning_rate": 1.6919344402680231e-07,
- "loss": 0.03015017881989479,
- "num_tokens": 1108318.0,
- "reward": 0.12559716403484344,
- "reward_std": 0.22310735285282135,
- "rewards/true_env_reward_fn/mean": 0.12559716403484344,
- "rewards/true_env_reward_fn/std": 0.22310735285282135,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2150432765483856,
+ "epoch": 6.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09289928525686264,
+ "kl": 9.474463149672374e-05,
+ "learning_rate": 3.230037691888162e-07,
+ "loss": 0.08772514015436172,
+ "num_tokens": 6636101.0,
+ "reward": 0.6059033870697021,
+ "reward_std": 0.23812197148799896,
+ "rewards/true_env_reward_fn/mean": 0.6059033274650574,
+ "rewards/true_env_reward_fn/std": 0.23812197148799896,
"step": 269,
- "step_time": 4.379851057999986
+ "step_time": 8.47666211500109
},
{
"clip_ratio/high_max": 0.0,
@@ -7548,26 +7548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.2916911840438843,
- "epoch": 2.1951219512195124,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18381677567958832,
- "kl": 2.593698127384414e-05,
- "learning_rate": 1.6611244871038116e-07,
- "loss": 0.025029506534337997,
- "num_tokens": 1116025.0,
- "reward": 0.05200198292732239,
- "reward_std": 0.276480108499527,
- "rewards/true_env_reward_fn/mean": 0.05200198292732239,
- "rewards/true_env_reward_fn/std": 0.276480108499527,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.2745259404182434,
+ "epoch": 6.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09368392825126648,
+ "kl": 0.0001518711451353738,
+ "learning_rate": 3.1902897305644093e-07,
+ "loss": -0.003839995712041855,
+ "num_tokens": 6659877.0,
+ "reward": 0.5029901266098022,
+ "reward_std": 0.3467065095901489,
+ "rewards/true_env_reward_fn/mean": 0.5029900670051575,
+ "rewards/true_env_reward_fn/std": 0.3467065393924713,
"step": 270,
- "step_time": 3.6788300769985653
+ "step_time": 11.298448464000103
},
{
"clip_ratio/high_max": 0.0,
@@ -7576,26 +7576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 215.0,
- "completions/max_terminated_length": 215.0,
- "completions/mean_length": 94.625,
- "completions/mean_terminated_length": 94.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.2625537514686584,
- "epoch": 2.203252032520325,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.09628148376941681,
- "kl": 2.636932003952097e-05,
- "learning_rate": 1.6305239584189344e-07,
- "loss": 0.009904414415359497,
- "num_tokens": 1120434.0,
- "reward": 0.5924437046051025,
- "reward_std": 0.39917245507240295,
- "rewards/true_env_reward_fn/mean": 0.5924437046051025,
- "rewards/true_env_reward_fn/std": 0.39917245507240295,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 67.0625,
+ "completions/mean_terminated_length": 67.0625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.238816499710083,
+ "epoch": 6.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05819084122776985,
+ "kl": 6.655609695371822e-05,
+ "learning_rate": 3.150672944955818e-07,
+ "loss": -0.02771003544330597,
+ "num_tokens": 6679776.0,
+ "reward": 0.5772091746330261,
+ "reward_std": 0.17815756797790527,
+ "rewards/true_env_reward_fn/mean": 0.5772091746330261,
+ "rewards/true_env_reward_fn/std": 0.17815756797790527,
"step": 271,
- "step_time": 9.09279120499923
+ "step_time": 7.902968623000561
},
{
"clip_ratio/high_max": 0.0,
@@ -7604,26 +7604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.2491654753684998,
- "epoch": 2.2113821138211383,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015083610196597874,
- "kl": 1.4522283436235739e-05,
- "learning_rate": 1.6001355942052182e-07,
- "loss": 7.247089683914965e-07,
- "num_tokens": 1122349.0,
- "reward": 0.812765896320343,
- "reward_std": 0.04731824994087219,
- "rewards/true_env_reward_fn/mean": 0.812765896320343,
- "rewards/true_env_reward_fn/std": 0.04731824994087219,
+ "completions/max_length": 145.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 75.10417175292969,
+ "completions/mean_terminated_length": 75.10417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3645851016044617,
+ "epoch": 6.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07474905252456665,
+ "kl": 7.497054593841312e-05,
+ "learning_rate": 3.1111902066602724e-07,
+ "loss": 0.07271203398704529,
+ "num_tokens": 6704717.0,
+ "reward": 0.42504438757896423,
+ "reward_std": 0.2500284016132355,
+ "rewards/true_env_reward_fn/mean": 0.42504438757896423,
+ "rewards/true_env_reward_fn/std": 0.2500284016132355,
"step": 272,
- "step_time": 3.7730076539992297
+ "step_time": 11.259095110999624
},
{
"clip_ratio/high_max": 0.0,
@@ -7632,26 +7632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 71.25,
- "completions/mean_terminated_length": 71.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1462301015853882,
- "epoch": 2.2195121951219514,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00012917000276502222,
- "kl": 1.3649782886204775e-05,
- "learning_rate": 1.569962115457138e-07,
- "loss": 6.695274805679219e-07,
- "num_tokens": 1124831.0,
- "reward": 0.7329437732696533,
- "reward_std": 0.22123214602470398,
- "rewards/true_env_reward_fn/mean": 0.7329437732696533,
- "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 67.64583587646484,
+ "completions/mean_terminated_length": 67.64583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2410458326339722,
+ "epoch": 6.658536585365853,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0750170573592186,
+ "kl": 8.587932643422391e-05,
+ "learning_rate": 3.0718443775593225e-07,
+ "loss": -0.019169267266988754,
+ "num_tokens": 6727060.0,
+ "reward": 0.5114523768424988,
+ "reward_std": 0.17165428400039673,
+ "rewards/true_env_reward_fn/mean": 0.5114523768424988,
+ "rewards/true_env_reward_fn/std": 0.17165428400039673,
"step": 273,
- "step_time": 3.907510233000721
+ "step_time": 11.097374408000178
},
{
"clip_ratio/high_max": 0.0,
@@ -7660,26 +7660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 175.0,
- "completions/max_terminated_length": 175.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2922418713569641,
- "epoch": 2.227642276422764,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15563185513019562,
- "kl": 3.604595076467376e-05,
- "learning_rate": 1.5400062239281858e-07,
- "loss": -0.034219659864902496,
- "num_tokens": 1128837.0,
- "reward": 0.46255773305892944,
- "reward_std": 0.24893923103809357,
- "rewards/true_env_reward_fn/mean": 0.46255773305892944,
- "rewards/true_env_reward_fn/std": 0.24893923103809357,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 67.08333587646484,
+ "completions/mean_terminated_length": 67.08333587646484,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2417791783809662,
+ "epoch": 6.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08969064801931381,
+ "kl": 5.61167857995315e-05,
+ "learning_rate": 3.0326383096107423e-07,
+ "loss": 0.11341358721256256,
+ "num_tokens": 6752824.0,
+ "reward": 0.4772833287715912,
+ "reward_std": 0.31618404388427734,
+ "rewards/true_env_reward_fn/mean": 0.4772833287715912,
+ "rewards/true_env_reward_fn/std": 0.31618407368659973,
"step": 274,
- "step_time": 7.502110859999448
+ "step_time": 11.189890726001067
},
{
"clip_ratio/high_max": 0.0,
@@ -7688,26 +7688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.2709790468215942,
- "epoch": 2.2357723577235773,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.47917175292969,
+ "completions/mean_terminated_length": 67.47917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.272821843624115,
+ "epoch": 6.7073170731707314,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.24958185851573944,
- "kl": 6.180045966175385e-05,
- "learning_rate": 1.5102706018889428e-07,
- "loss": -0.10148808360099792,
- "num_tokens": 1132891.0,
- "reward": 0.4240284562110901,
- "reward_std": 0.43752968311309814,
- "rewards/true_env_reward_fn/mean": 0.4240284562110901,
- "rewards/true_env_reward_fn/std": 0.43752965331077576,
+ "grad_norm": 0.08874963968992233,
+ "kl": 0.00015055539006425533,
+ "learning_rate": 2.9935748446418065e-07,
+ "loss": 0.02028803899884224,
+ "num_tokens": 6776703.0,
+ "reward": 0.49093031883239746,
+ "reward_std": 0.25163188576698303,
+ "rewards/true_env_reward_fn/mean": 0.49093031883239746,
+ "rewards/true_env_reward_fn/std": 0.25163188576698303,
"step": 275,
- "step_time": 3.251475233999372
+ "step_time": 9.941926390000845
},
{
"clip_ratio/high_max": 0.0,
@@ -7716,26 +7716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 55.5,
- "completions/mean_terminated_length": 55.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.2637454867362976,
- "epoch": 2.2439024390243905,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.439944602083415e-05,
- "kl": 1.127877567341784e-05,
- "learning_rate": 1.4807579118869146e-07,
- "loss": 5.64579522688291e-07,
- "num_tokens": 1137611.0,
- "reward": 0.4544333219528198,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4544333219528198,
- "rewards/true_env_reward_fn/std": 0.0,
+ "completions/max_length": 365.0,
+ "completions/max_terminated_length": 365.0,
+ "completions/mean_length": 68.04167175292969,
+ "completions/mean_terminated_length": 68.04167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0912510752677917,
+ "epoch": 6.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060312703251838684,
+ "kl": 6.840269179519964e-05,
+ "learning_rate": 2.9546568141433e-07,
+ "loss": -0.036469291895627975,
+ "num_tokens": 6799417.0,
+ "reward": 0.5543485879898071,
+ "reward_std": 0.22147472202777863,
+ "rewards/true_env_reward_fn/mean": 0.5543485283851624,
+ "rewards/true_env_reward_fn/std": 0.22147469222545624,
"step": 276,
- "step_time": 3.311975311999049
+ "step_time": 22.291117544999906
},
{
"clip_ratio/high_max": 0.0,
@@ -7744,26 +7744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.1065265536308289,
- "epoch": 2.252032520325203,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13068579137325287,
- "kl": 4.4293181417742744e-05,
- "learning_rate": 1.4514707965081262e-07,
- "loss": 0.04848391190171242,
- "num_tokens": 1142087.0,
- "reward": 0.2852628827095032,
- "reward_std": 0.23009054362773895,
- "rewards/true_env_reward_fn/mean": 0.2852628827095032,
- "rewards/true_env_reward_fn/std": 0.23009057343006134,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.895835876464844,
+ "completions/mean_terminated_length": 59.895835876464844,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.2566059231758118,
+ "epoch": 6.7560975609756095,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09982399642467499,
+ "kl": 0.00020091429905733094,
+ "learning_rate": 2.9158870390642863e-07,
+ "loss": -0.005738064646720886,
+ "num_tokens": 6829804.0,
+ "reward": 0.24754562973976135,
+ "reward_std": 0.23038579523563385,
+ "rewards/true_env_reward_fn/mean": 0.24754561483860016,
+ "rewards/true_env_reward_fn/std": 0.23038578033447266,
"step": 277,
- "step_time": 3.9521008399988204
+ "step_time": 9.732460060999983
},
{
"clip_ratio/high_max": 0.0,
@@ -7772,26 +7772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 168.0,
- "completions/max_terminated_length": 168.0,
- "completions/mean_length": 104.5,
- "completions/mean_terminated_length": 104.5,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.5374161005020142,
- "epoch": 2.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.129844531416893,
- "kl": 3.9368313082377426e-05,
- "learning_rate": 1.4224118781404917e-07,
- "loss": -0.1482687145471573,
- "num_tokens": 1147731.0,
- "reward": 0.43388551473617554,
- "reward_std": 0.39525240659713745,
- "rewards/true_env_reward_fn/mean": 0.43388551473617554,
- "rewards/true_env_reward_fn/std": 0.39525243639945984,
+ "completions/max_length": 149.0,
+ "completions/max_terminated_length": 149.0,
+ "completions/mean_length": 71.60417175292969,
+ "completions/mean_terminated_length": 71.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2196767628192902,
+ "epoch": 6.780487804878049,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03178449347615242,
+ "kl": 5.627466816804372e-05,
+ "learning_rate": 2.8772683296076194e-07,
+ "loss": 0.002586783841252327,
+ "num_tokens": 6852617.0,
+ "reward": 0.5281299352645874,
+ "reward_std": 0.22268518805503845,
+ "rewards/true_env_reward_fn/mean": 0.5281298756599426,
+ "rewards/true_env_reward_fn/std": 0.22268518805503845,
"step": 278,
- "step_time": 7.55689369099855
+ "step_time": 11.629210506000163
},
{
"clip_ratio/high_max": 0.0,
@@ -7800,26 +7800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.0975646376609802,
- "epoch": 2.2682926829268295,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1335010528564453,
- "kl": 2.2682882445224095e-05,
- "learning_rate": 1.3935837587390214e-07,
- "loss": -0.009766265749931335,
- "num_tokens": 1151246.0,
- "reward": 0.37655720114707947,
- "reward_std": 0.37258440256118774,
- "rewards/true_env_reward_fn/mean": 0.37655720114707947,
- "rewards/true_env_reward_fn/std": 0.37258440256118774,
+ "completions/max_length": 169.0,
+ "completions/max_terminated_length": 169.0,
+ "completions/mean_length": 68.66667175292969,
+ "completions/mean_terminated_length": 68.66667175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1991542279720306,
+ "epoch": 6.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07567304372787476,
+ "kl": 0.00015181046910583973,
+ "learning_rate": 2.8388034850262646e-07,
+ "loss": -0.04906900227069855,
+ "num_tokens": 6881529.0,
+ "reward": 0.3906375765800476,
+ "reward_std": 0.3576822578907013,
+ "rewards/true_env_reward_fn/mean": 0.3906375467777252,
+ "rewards/true_env_reward_fn/std": 0.3576822876930237,
"step": 279,
- "step_time": 3.8792882219986495
+ "step_time": 18.253660386000774
},
{
"clip_ratio/high_max": 0.0,
@@ -7828,26 +7828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 63.5,
- "completions/mean_terminated_length": 63.5,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.1846895217895508,
- "epoch": 2.2764227642276422,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.138702854514122,
- "kl": 2.0143882466072682e-05,
- "learning_rate": 1.3649890195928254e-07,
- "loss": 0.013748884201049805,
- "num_tokens": 1153222.0,
- "reward": 0.8218116760253906,
- "reward_std": 0.1293545663356781,
- "rewards/true_env_reward_fn/mean": 0.8218116760253906,
- "rewards/true_env_reward_fn/std": 0.1293545812368393,
+ "completions/max_length": 208.0,
+ "completions/max_terminated_length": 208.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.263102412223816,
+ "epoch": 6.829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09104589372873306,
+ "kl": 0.00012209633769089123,
+ "learning_rate": 2.8004952934203837e-07,
+ "loss": 0.06642289459705353,
+ "num_tokens": 6905111.0,
+ "reward": 0.3988339304924011,
+ "reward_std": 0.34396088123321533,
+ "rewards/true_env_reward_fn/mean": 0.39883390069007874,
+ "rewards/true_env_reward_fn/std": 0.3439609110355377,
"step": 280,
- "step_time": 3.42550413599929
+ "step_time": 15.519001798998943
},
{
"clip_ratio/high_max": 0.0,
@@ -7856,26 +7856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 64.0,
- "completions/mean_terminated_length": 64.0,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3451175689697266,
- "epoch": 2.2845528455284554,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1508527249097824,
- "kl": 1.6968931049632374e-05,
- "learning_rate": 1.336630221093991e-07,
- "loss": 0.04001428931951523,
- "num_tokens": 1157606.0,
- "reward": 0.4994586706161499,
- "reward_std": 0.10593737661838531,
- "rewards/true_env_reward_fn/mean": 0.4994586706161499,
- "rewards/true_env_reward_fn/std": 0.10593737661838531,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 59.0625,
+ "completions/mean_terminated_length": 59.0625,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.208267629146576,
+ "epoch": 6.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07465670257806778,
+ "kl": 7.14320785846212e-05,
+ "learning_rate": 2.762346531535246e-07,
+ "loss": 0.04124641790986061,
+ "num_tokens": 6926634.0,
+ "reward": 0.5870868563652039,
+ "reward_std": 0.28217118978500366,
+ "rewards/true_env_reward_fn/mean": 0.5870868563652039,
+ "rewards/true_env_reward_fn/std": 0.2821711599826813,
"step": 281,
- "step_time": 3.914840199000537
+ "step_time": 11.224198447001072
},
{
"clip_ratio/high_max": 0.0,
@@ -7884,26 +7884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2436452507972717,
- "epoch": 2.292682926829268,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12380471080541611,
- "kl": 4.350653580331709e-05,
- "learning_rate": 1.3085099025083245e-07,
- "loss": -0.029160797595977783,
- "num_tokens": 1162012.0,
- "reward": 0.5066306591033936,
- "reward_std": 0.28914663195610046,
- "rewards/true_env_reward_fn/mean": 0.5066306591033936,
- "rewards/true_env_reward_fn/std": 0.28914666175842285,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.02083587646484,
+ "completions/mean_terminated_length": 72.02083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3142951428890228,
+ "epoch": 6.878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07364725321531296,
+ "kl": 0.00011016946336894762,
+ "learning_rate": 2.7243599645599574e-07,
+ "loss": 0.004198473412543535,
+ "num_tokens": 6960219.0,
+ "reward": 0.34401482343673706,
+ "reward_std": 0.24699951708316803,
+ "rewards/true_env_reward_fn/mean": 0.3440147936344147,
+ "rewards/true_env_reward_fn/std": 0.24699951708316803,
"step": 282,
- "step_time": 3.9093819319987233
+ "step_time": 15.210776117999558
},
{
"clip_ratio/high_max": 0.0,
@@ -7912,26 +7912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 59.375,
- "completions/mean_terminated_length": 59.375,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.10621577501297,
- "epoch": 2.3008130081300813,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001239393459400162,
- "kl": 1.6069413049990544e-05,
- "learning_rate": 1.2806305817479771e-07,
- "loss": 8.174432082341809e-07,
- "num_tokens": 1165367.0,
- "reward": 0.6557307243347168,
- "reward_std": 0.2151959389448166,
- "rewards/true_env_reward_fn/mean": 0.6557307243347168,
- "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 60.583335876464844,
+ "completions/mean_terminated_length": 60.583335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2013934552669525,
+ "epoch": 6.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07040710747241974,
+ "kl": 7.988750212462037e-05,
+ "learning_rate": 2.686538345927026e-07,
+ "loss": 0.06334929168224335,
+ "num_tokens": 6982879.0,
+ "reward": 0.5139331221580505,
+ "reward_std": 0.19786590337753296,
+ "rewards/true_env_reward_fn/mean": 0.5139331221580505,
+ "rewards/true_env_reward_fn/std": 0.19786591827869415,
"step": 283,
- "step_time": 3.8117841049988783
+ "step_time": 10.240000448000501
},
{
"clip_ratio/high_max": 0.0,
@@ -7940,26 +7940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 66.0,
- "completions/mean_terminated_length": 66.0,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.382466197013855,
- "epoch": 2.3089430894308944,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1446879506111145,
- "kl": 3.576014205464162e-05,
- "learning_rate": 1.2529947551459964e-07,
- "loss": -0.04731176793575287,
- "num_tokens": 1169243.0,
- "reward": 0.496622234582901,
- "reward_std": 0.17860308289527893,
- "rewards/true_env_reward_fn/mean": 0.496622234582901,
- "rewards/true_env_reward_fn/std": 0.17860306799411774,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2223551571369171,
+ "epoch": 6.926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07971613109111786,
+ "kl": 0.00016717875041649677,
+ "learning_rate": 2.64888441711279e-07,
+ "loss": 0.03706979751586914,
+ "num_tokens": 7003990.0,
+ "reward": 0.5301157832145691,
+ "reward_std": 0.2464885115623474,
+ "rewards/true_env_reward_fn/mean": 0.5301157832145691,
+ "rewards/true_env_reward_fn/std": 0.2464885115623474,
"step": 284,
- "step_time": 3.649135475998264
+ "step_time": 9.309556909001913
},
{
"clip_ratio/high_max": 0.0,
@@ -7968,26 +7968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 151.0,
- "completions/max_terminated_length": 151.0,
- "completions/mean_length": 61.375,
- "completions/mean_terminated_length": 61.375,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1946157813072205,
- "epoch": 2.317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10612626373767853,
- "kl": 2.0257402866263874e-05,
- "learning_rate": 1.2256048972327967e-07,
- "loss": -0.029511645436286926,
- "num_tokens": 1173594.0,
- "reward": 0.5235810279846191,
- "reward_std": 0.2810492217540741,
- "rewards/true_env_reward_fn/mean": 0.5235810279846191,
- "rewards/true_env_reward_fn/std": 0.2810492217540741,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.309690535068512,
+ "epoch": 6.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09541535377502441,
+ "kl": 0.00014337312313728034,
+ "learning_rate": 2.6114009074386844e-07,
+ "loss": -0.04744558781385422,
+ "num_tokens": 7027733.0,
+ "reward": 0.468932569026947,
+ "reward_std": 0.2396899163722992,
+ "rewards/true_env_reward_fn/mean": 0.46893253922462463,
+ "rewards/true_env_reward_fn/std": 0.2396899312734604,
"step": 285,
- "step_time": 6.7011265000001
+ "step_time": 8.91306197799986
},
{
"clip_ratio/high_max": 0.0,
@@ -7996,26 +7996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 68.75,
- "completions/mean_terminated_length": 68.75,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2879216074943542,
- "epoch": 2.3252032520325203,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12234477698802948,
- "kl": 1.4488860415440286e-05,
- "learning_rate": 1.1984634605145978e-07,
- "loss": -0.05451745539903641,
- "num_tokens": 1176384.0,
- "reward": 0.6496104001998901,
- "reward_std": 0.3368098735809326,
- "rewards/true_env_reward_fn/mean": 0.6496104001998901,
- "rewards/true_env_reward_fn/std": 0.3368098735809326,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1709823310375214,
+ "epoch": 6.975609756097561,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048187412321567535,
+ "kl": 9.718035107653122e-05,
+ "learning_rate": 2.5740905338734306e-07,
+ "loss": 0.010684527456760406,
+ "num_tokens": 7058137.0,
+ "reward": 0.44316989183425903,
+ "reward_std": 0.14355739951133728,
+ "rewards/true_env_reward_fn/mean": 0.44316986203193665,
+ "rewards/true_env_reward_fn/std": 0.14355739951133728,
"step": 286,
- "step_time": 4.18911992899848
+ "step_time": 9.546786461999545
},
{
"clip_ratio/high_max": 0.0,
@@ -8024,26 +8024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 105.0,
- "completions/max_terminated_length": 105.0,
- "completions/mean_length": 70.875,
- "completions/mean_terminated_length": 70.875,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.0649144053459167,
- "epoch": 2.3333333333333335,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1467445194721222,
- "kl": 3.947542245441582e-05,
- "learning_rate": 1.17157287525381e-07,
- "loss": -0.029090911149978638,
- "num_tokens": 1179651.0,
- "reward": 0.4754716753959656,
- "reward_std": 0.2500669062137604,
- "rewards/true_env_reward_fn/mean": 0.4754716753959656,
- "rewards/true_env_reward_fn/std": 0.2500669062137604,
+ "completions/max_length": 458.0,
+ "completions/max_terminated_length": 458.0,
+ "completions/mean_length": 77.97917175292969,
+ "completions/mean_terminated_length": 77.97917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3698437809944153,
+ "epoch": 7.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05576475337147713,
+ "kl": 9.964485252567101e-05,
+ "learning_rate": 2.536956000836082e-07,
+ "loss": 0.08634226769208908,
+ "num_tokens": 7082840.0,
+ "reward": 0.5030691027641296,
+ "reward_std": 0.2275215983390808,
+ "rewards/true_env_reward_fn/mean": 0.5030691027641296,
+ "rewards/true_env_reward_fn/std": 0.22752158343791962,
"step": 287,
- "step_time": 4.5810332049986755
+ "step_time": 26.703723129000537
},
{
"clip_ratio/high_max": 0.0,
@@ -8052,26 +8052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 1.002779871225357,
- "epoch": 2.341463414634146,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12533478438854218,
- "kl": 2.3054610210238025e-05,
- "learning_rate": 1.1449355492514437e-07,
- "loss": -0.04500773549079895,
- "num_tokens": 1181547.0,
- "reward": 0.7334807515144348,
- "reward_std": 0.12557923793792725,
- "rewards/true_env_reward_fn/mean": 0.7334807515144348,
- "rewards/true_env_reward_fn/std": 0.12557923793792725,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2532718479633331,
+ "epoch": 7.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08579511940479279,
+ "kl": 9.319775290350663e-05,
+ "learning_rate": 2.500000000000001e-07,
+ "loss": -0.013113420456647873,
+ "num_tokens": 7098689.0,
+ "reward": 0.5958684682846069,
+ "reward_std": 0.1863170713186264,
+ "rewards/true_env_reward_fn/mean": 0.5958684682846069,
+ "rewards/true_env_reward_fn/std": 0.1863170564174652,
"step": 288,
- "step_time": 3.8661079009998502
+ "step_time": 7.618657231000725
},
{
"clip_ratio/high_max": 0.0,
@@ -8080,26 +8080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 294.0,
- "completions/max_terminated_length": 294.0,
- "completions/mean_length": 81.25,
- "completions/mean_terminated_length": 81.25,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2280530333518982,
- "epoch": 2.3495934959349594,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23682235181331635,
- "kl": 5.290110129863024e-05,
- "learning_rate": 1.1185538676315052e-07,
- "loss": 0.5538168549537659,
- "num_tokens": 1183973.0,
- "reward": 0.5785378217697144,
- "reward_std": 0.29215970635414124,
- "rewards/true_env_reward_fn/mean": 0.5785378217697144,
- "rewards/true_env_reward_fn/std": 0.2921597361564636,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.20833587646484,
+ "completions/mean_terminated_length": 68.20833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.1685407161712646,
+ "epoch": 7.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07076086848974228,
+ "kl": 7.435419411194744e-05,
+ "learning_rate": 2.4632252100977564e-07,
+ "loss": 0.010196422226727009,
+ "num_tokens": 7126387.0,
+ "reward": 0.47137731313705444,
+ "reward_std": 0.20232117176055908,
+ "rewards/true_env_reward_fn/mean": 0.47137728333473206,
+ "rewards/true_env_reward_fn/std": 0.20232117176055908,
"step": 289,
- "step_time": 11.744046860998424
+ "step_time": 10.741382757999418
},
{
"clip_ratio/high_max": 0.0,
@@ -8108,26 +8108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.264986515045166,
- "epoch": 2.3577235772357725,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001335782726528123,
- "kl": 1.7416054106433876e-05,
- "learning_rate": 1.0924301926274248e-07,
- "loss": 8.602528396295384e-07,
- "num_tokens": 1187311.0,
- "reward": 0.4899469017982483,
- "reward_std": 0.24049179255962372,
- "rewards/true_env_reward_fn/mean": 0.4899469017982483,
- "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2131870985031128,
+ "epoch": 7.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07972414791584015,
+ "kl": 0.00010809541072376305,
+ "learning_rate": 2.426634296726955e-07,
+ "loss": 0.07707639783620834,
+ "num_tokens": 7150947.0,
+ "reward": 0.4951697587966919,
+ "reward_std": 0.31705158948898315,
+ "rewards/true_env_reward_fn/mean": 0.4951697587966919,
+ "rewards/true_env_reward_fn/std": 0.31705158948898315,
"step": 290,
- "step_time": 4.003069795000556
+ "step_time": 11.908707627000695
},
{
"clip_ratio/high_max": 0.0,
@@ -8136,26 +8136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 109.0,
- "completions/max_terminated_length": 109.0,
- "completions/mean_length": 65.875,
- "completions/mean_terminated_length": 65.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.222926139831543,
- "epoch": 2.3658536585365852,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.209110481198877e-05,
- "kl": 1.1452370472397888e-05,
- "learning_rate": 1.0665668633705572e-07,
- "loss": 5.764911747974111e-07,
- "num_tokens": 1191370.0,
- "reward": 0.5930472612380981,
- "reward_std": 0.14818456768989563,
- "rewards/true_env_reward_fn/mean": 0.5930472612380981,
- "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 66.41667175292969,
+ "completions/mean_terminated_length": 66.41667175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2247934639453888,
+ "epoch": 7.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06208934634923935,
+ "kl": 8.92497146196547e-05,
+ "learning_rate": 2.390229912157033e-07,
+ "loss": 0.021278446540236473,
+ "num_tokens": 7174671.0,
+ "reward": 0.44533461332321167,
+ "reward_std": 0.20755091309547424,
+ "rewards/true_env_reward_fn/mean": 0.4453345835208893,
+ "rewards/true_env_reward_fn/std": 0.20755092799663544,
"step": 291,
- "step_time": 4.77749846199913
+ "step_time": 11.859711304000484
},
{
"clip_ratio/high_max": 0.0,
@@ -8164,26 +8164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 65.625,
- "completions/mean_terminated_length": 65.625,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "entropy": 1.313057780265808,
- "epoch": 2.3739837398373984,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.671779556199908e-05,
- "kl": 1.1384066056052689e-05,
- "learning_rate": 1.0409661956807174e-07,
- "loss": 5.689008162335085e-07,
- "num_tokens": 1195851.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.25,
+ "completions/mean_terminated_length": 66.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3344177305698395,
+ "epoch": 7.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08249509334564209,
+ "kl": 0.00013599474277725676,
+ "learning_rate": 2.3540146951369966e-07,
+ "loss": 0.05746981129050255,
+ "num_tokens": 7202291.0,
+ "reward": 0.36803489923477173,
+ "reward_std": 0.22159868478775024,
+ "rewards/true_env_reward_fn/mean": 0.36803486943244934,
+ "rewards/true_env_reward_fn/std": 0.22159868478775024,
"step": 292,
- "step_time": 3.4948791150000034
+ "step_time": 9.794866193998132
},
{
"clip_ratio/high_max": 0.0,
@@ -8192,26 +8192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.3050158619880676,
- "epoch": 2.3821138211382116,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15694627165794373,
- "kl": 2.6891467314271722e-05,
- "learning_rate": 1.0156304818588308e-07,
- "loss": -0.044491954147815704,
- "num_tokens": 1201316.0,
- "reward": 0.18382371962070465,
- "reward_std": 0.18414245545864105,
- "rewards/true_env_reward_fn/mean": 0.18382371962070465,
- "rewards/true_env_reward_fn/std": 0.18414245545864105,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 64.02083587646484,
+ "completions/mean_terminated_length": 64.02083587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2605153024196625,
+ "epoch": 7.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07226194441318512,
+ "kl": 7.86567807153915e-05,
+ "learning_rate": 2.3179912707041666e-07,
+ "loss": -0.06701754778623581,
+ "num_tokens": 7221060.0,
+ "reward": 0.6259939670562744,
+ "reward_std": 0.30331701040267944,
+ "rewards/true_env_reward_fn/mean": 0.6259939074516296,
+ "rewards/true_env_reward_fn/std": 0.30331701040267944,
"step": 293,
- "step_time": 3.900356202000694
+ "step_time": 13.536596455999643
},
{
"clip_ratio/high_max": 0.0,
@@ -8220,26 +8220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 52.25,
- "completions/mean_terminated_length": 52.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2079132199287415,
- "epoch": 2.3902439024390243,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.25304585695266724,
- "kl": 4.250183792464668e-05,
- "learning_rate": 9.905619904816749e-08,
- "loss": 0.008140146732330322,
- "num_tokens": 1204090.0,
- "reward": 0.43626630306243896,
- "reward_std": 0.3027261197566986,
- "rewards/true_env_reward_fn/mean": 0.43626630306243896,
- "rewards/true_env_reward_fn/std": 0.3027261197566986,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 76.83333587646484,
+ "completions/mean_terminated_length": 76.83333587646484,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.275952398777008,
+ "epoch": 7.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07967559248209,
+ "kl": 0.0001376894815621199,
+ "learning_rate": 2.2821622499938948e-07,
+ "loss": -0.024285754188895226,
+ "num_tokens": 7251156.0,
+ "reward": 0.3353421688079834,
+ "reward_std": 0.26816248893737793,
+ "rewards/true_env_reward_fn/mean": 0.3353421688079834,
+ "rewards/true_env_reward_fn/std": 0.26816248893737793,
"step": 294,
- "step_time": 3.1466946830005327
+ "step_time": 19.762229363001097
},
{
"clip_ratio/high_max": 0.0,
@@ -8248,26 +8248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2324314713478088,
- "epoch": 2.3983739837398375,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011851672024931759,
- "kl": 1.4868088328512385e-05,
- "learning_rate": 9.657629661987531e-08,
- "loss": 7.534490578109398e-07,
- "num_tokens": 1206449.0,
- "reward": 0.7232838273048401,
- "reward_std": 0.008955853059887886,
- "rewards/true_env_reward_fn/mean": 0.7232838273048401,
- "rewards/true_env_reward_fn/std": 0.008955853059887886,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.4140338003635406,
+ "epoch": 7.195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07337357848882675,
+ "kl": 0.00012654263991862535,
+ "learning_rate": 2.2465302300503008e-07,
+ "loss": -0.04723845049738884,
+ "num_tokens": 7277361.0,
+ "reward": 0.4155184030532837,
+ "reward_std": 0.23990045487880707,
+ "rewards/true_env_reward_fn/mean": 0.4155184030532837,
+ "rewards/true_env_reward_fn/std": 0.23990046977996826,
"step": 295,
- "step_time": 3.5492840760016406
+ "step_time": 11.263231479000751
},
{
"clip_ratio/high_max": 0.0,
@@ -8276,26 +8276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 65.125,
- "completions/mean_terminated_length": 65.125,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.2110244631767273,
- "epoch": 2.40650406504065,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 63.9375,
+ "completions/mean_terminated_length": 63.9375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.22734934091568,
+ "epoch": 7.219512195121951,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.18513579666614532,
- "kl": 6.467000457632821e-05,
- "learning_rate": 9.412356295313019e-08,
- "loss": -0.008508354425430298,
- "num_tokens": 1211294.0,
- "reward": 0.38126999139785767,
- "reward_std": 0.22339044511318207,
- "rewards/true_env_reward_fn/mean": 0.38126999139785767,
- "rewards/true_env_reward_fn/std": 0.22339043021202087,
+ "grad_norm": 0.0908711850643158,
+ "kl": 0.00016623977535346057,
+ "learning_rate": 2.2110977936380287e-07,
+ "loss": -0.03445049002766609,
+ "num_tokens": 7298998.0,
+ "reward": 0.564518392086029,
+ "reward_std": 0.30392351746559143,
+ "rewards/true_env_reward_fn/mean": 0.564518392086029,
+ "rewards/true_env_reward_fn/std": 0.30392348766326904,
"step": 296,
- "step_time": 3.6129159619995335
+ "step_time": 10.9702629049998
},
{
"clip_ratio/high_max": 0.0,
@@ -8304,26 +8304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 58.75,
- "completions/mean_terminated_length": 58.75,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.1828523874282837,
- "epoch": 2.4146341463414633,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13915051519870758,
- "kl": 2.76857699645916e-05,
- "learning_rate": 9.169821766734668e-08,
- "loss": -0.019797056913375854,
- "num_tokens": 1214972.0,
- "reward": 0.5778937339782715,
- "reward_std": 0.4268997013568878,
- "rewards/true_env_reward_fn/mean": 0.5778937339782715,
- "rewards/true_env_reward_fn/std": 0.4268997013568878,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.312496393918991,
+ "epoch": 7.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08918203413486481,
+ "kl": 0.00018221777827420738,
+ "learning_rate": 2.1758675090550328e-07,
+ "loss": 0.023428799584507942,
+ "num_tokens": 7320725.0,
+ "reward": 0.4695018231868744,
+ "reward_std": 0.2527172565460205,
+ "rewards/true_env_reward_fn/mean": 0.4695018231868744,
+ "rewards/true_env_reward_fn/std": 0.2527172565460205,
"step": 297,
- "step_time": 3.5299333029997797
+ "step_time": 12.689384352000161
},
{
"clip_ratio/high_max": 0.0,
@@ -8332,26 +8332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2796242237091064,
- "epoch": 2.4227642276422765,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23225097358226776,
- "kl": 5.458398300106637e-05,
- "learning_rate": 8.930047792956585e-08,
- "loss": 0.028596192598342896,
- "num_tokens": 1221117.0,
- "reward": 0.09695600718259811,
- "reward_std": 0.23755072057247162,
- "rewards/true_env_reward_fn/mean": 0.09695600718259811,
- "rewards/true_env_reward_fn/std": 0.23755072057247162,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.260593295097351,
+ "epoch": 7.2682926829268295,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07724963873624802,
+ "kl": 4.775456409333856e-05,
+ "learning_rate": 2.1408419299464242e-07,
+ "loss": 0.03472111374139786,
+ "num_tokens": 7345397.0,
+ "reward": 0.39238378405570984,
+ "reward_std": 0.23648974299430847,
+ "rewards/true_env_reward_fn/mean": 0.39238378405570984,
+ "rewards/true_env_reward_fn/std": 0.23648977279663086,
"step": 298,
- "step_time": 3.9654863289997593
+ "step_time": 10.398283558999537
},
{
"clip_ratio/high_max": 0.0,
@@ -8360,26 +8360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.3131609559059143,
- "epoch": 2.430894308943089,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.15491950511932373,
- "kl": 2.3678386241954286e-05,
- "learning_rate": 8.693055843500867e-08,
- "loss": 0.08264091610908508,
- "num_tokens": 1226670.0,
- "reward": 0.22746901214122772,
- "reward_std": 0.2765822112560272,
- "rewards/true_env_reward_fn/mean": 0.22746901214122772,
- "rewards/true_env_reward_fn/std": 0.2765822112560272,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.272550791501999,
+ "epoch": 7.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0737687349319458,
+ "kl": 7.711273838140187e-05,
+ "learning_rate": 2.1060235951193578e-07,
+ "loss": 0.010874807834625244,
+ "num_tokens": 7371231.0,
+ "reward": 0.4486481845378876,
+ "reward_std": 0.26590272784233093,
+ "rewards/true_env_reward_fn/mean": 0.4486481845378876,
+ "rewards/true_env_reward_fn/std": 0.26590269804000854,
"step": 299,
- "step_time": 3.9300464680000005
+ "step_time": 12.961759718001304
},
{
"clip_ratio/high_max": 0.0,
@@ -8388,26 +8388,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 61.75,
- "completions/mean_terminated_length": 61.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.113481342792511,
- "epoch": 2.4390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14261895418167114,
- "kl": 2.8488028874562588e-05,
- "learning_rate": 8.458867138785369e-08,
- "loss": 0.012870386242866516,
- "num_tokens": 1230460.0,
- "reward": 0.6178936958312988,
- "reward_std": 0.3830615282058716,
- "rewards/true_env_reward_fn/mean": 0.6178936958312988,
- "rewards/true_env_reward_fn/std": 0.3830614984035492,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3426357805728912,
+ "epoch": 7.317073170731708,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09169661998748779,
+ "kl": 0.00015296797573682852,
+ "learning_rate": 2.071415028359026e-07,
+ "loss": -0.0708877444267273,
+ "num_tokens": 7397151.0,
+ "reward": 0.5012298822402954,
+ "reward_std": 0.21363919973373413,
+ "rewards/true_env_reward_fn/mean": 0.5012298226356506,
+ "rewards/true_env_reward_fn/std": 0.21363921463489532,
"step": 300,
- "step_time": 4.004705740000645
+ "step_time": 13.761256955000135
},
{
"clip_ratio/high_max": 0.0,
@@ -8416,26 +8416,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 54.25,
- "completions/mean_terminated_length": 54.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.2633118629455566,
- "epoch": 2.4471544715447155,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1559160053730011,
- "kl": 3.0400691230170196e-05,
- "learning_rate": 8.227502648223494e-08,
- "loss": 0.023099511861801147,
- "num_tokens": 1234850.0,
- "reward": 0.3328326642513275,
- "reward_std": 0.2606535255908966,
- "rewards/true_env_reward_fn/mean": 0.3328326642513275,
- "rewards/true_env_reward_fn/std": 0.260653555393219,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.75,
+ "completions/mean_terminated_length": 70.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2883423566818237,
+ "epoch": 7.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06652592122554779,
+ "kl": 0.00011371383516234346,
+ "learning_rate": 2.0370187382457066e-07,
+ "loss": -0.011642830446362495,
+ "num_tokens": 7426403.0,
+ "reward": 0.3339906632900238,
+ "reward_std": 0.20955638587474823,
+ "rewards/true_env_reward_fn/mean": 0.3339906632900238,
+ "rewards/true_env_reward_fn/std": 0.20955640077590942,
"step": 301,
- "step_time": 4.035395368000536
+ "step_time": 9.628323140000248
},
{
"clip_ratio/high_max": 0.0,
@@ -8444,26 +8444,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 154.0,
- "completions/max_terminated_length": 154.0,
- "completions/mean_length": 79.875,
- "completions/mean_terminated_length": 79.875,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4545465111732483,
- "epoch": 2.4552845528455283,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13911883533000946,
- "kl": 5.561073703574948e-05,
- "learning_rate": 7.99898308834662e-08,
- "loss": 0.10435273498296738,
- "num_tokens": 1237629.0,
- "reward": 0.3929310441017151,
- "reward_std": 0.294041246175766,
- "rewards/true_env_reward_fn/mean": 0.3929310441017151,
- "rewards/true_env_reward_fn/std": 0.294041246175766,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 68.02083587646484,
+ "completions/mean_terminated_length": 68.02083587646484,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.1677636206150055,
+ "epoch": 7.365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08921164274215698,
+ "kl": 7.750577606202569e-05,
+ "learning_rate": 2.0028372179729402e-07,
+ "loss": 0.024114318192005157,
+ "num_tokens": 7458380.0,
+ "reward": 0.3653101921081543,
+ "reward_std": 0.3042241334915161,
+ "rewards/true_env_reward_fn/mean": 0.3653101921081543,
+ "rewards/true_env_reward_fn/std": 0.3042241334915161,
"step": 302,
- "step_time": 6.382147416999942
+ "step_time": 17.152215452000746
},
{
"clip_ratio/high_max": 0.0,
@@ -8472,26 +8472,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 1.4197523593902588,
- "epoch": 2.4634146341463414,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11274738609790802,
- "kl": 1.6755700926296413e-05,
- "learning_rate": 7.773328920949151e-08,
- "loss": -0.1040230244398117,
- "num_tokens": 1240738.0,
- "reward": 0.6045305728912354,
- "reward_std": 0.10337947309017181,
- "rewards/true_env_reward_fn/mean": 0.6045305728912354,
- "rewards/true_env_reward_fn/std": 0.10337948054075241,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.04166793823242,
+ "completions/mean_terminated_length": 61.04166793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1981053352355957,
+ "epoch": 7.390243902439025,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08189807087182999,
+ "kl": 0.00016033334395615384,
+ "learning_rate": 1.9688729451668111e-07,
+ "loss": 0.017046045511960983,
+ "num_tokens": 7481590.0,
+ "reward": 0.5003601312637329,
+ "reward_std": 0.3917968273162842,
+ "rewards/true_env_reward_fn/mean": 0.5003601312637329,
+ "rewards/true_env_reward_fn/std": 0.3917968273162842,
"step": 303,
- "step_time": 3.5754013399982796
+ "step_time": 10.428820308000468
},
{
"clip_ratio/high_max": 0.0,
@@ -8500,26 +8500,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 105.0,
- "completions/max_terminated_length": 105.0,
- "completions/mean_length": 69.625,
- "completions/mean_terminated_length": 69.625,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.3458821177482605,
- "epoch": 2.4715447154471546,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11526551097631454,
- "kl": 2.5664294298621826e-05,
- "learning_rate": 7.550560351256309e-08,
- "loss": 0.0017936527729034424,
- "num_tokens": 1245803.0,
- "reward": 0.2295326590538025,
- "reward_std": 0.3014034032821655,
- "rewards/true_env_reward_fn/mean": 0.2295326590538025,
- "rewards/true_env_reward_fn/std": 0.3014034032821655,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 64.95833587646484,
+ "completions/mean_terminated_length": 64.95833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2136133313179016,
+ "epoch": 7.414634146341464,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07148941606283188,
+ "kl": 5.305510785547085e-05,
+ "learning_rate": 1.9351283817063546e-07,
+ "loss": -0.009052902460098267,
+ "num_tokens": 7508932.0,
+ "reward": 0.3940971791744232,
+ "reward_std": 0.2546152174472809,
+ "rewards/true_env_reward_fn/mean": 0.3940971791744232,
+ "rewards/true_env_reward_fn/std": 0.25461524724960327,
"step": 304,
- "step_time": 4.8661928239998815
+ "step_time": 12.12407543900099
},
{
"clip_ratio/high_max": 0.0,
@@ -8528,26 +8528,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2587900161743164,
- "epoch": 2.4796747967479673,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1804642677307129,
- "kl": 3.644101343525108e-05,
- "learning_rate": 7.330697326114972e-08,
- "loss": 0.006622403860092163,
- "num_tokens": 1250224.0,
- "reward": 0.392159104347229,
- "reward_std": 0.23998720943927765,
- "rewards/true_env_reward_fn/mean": 0.392159104347229,
- "rewards/true_env_reward_fn/std": 0.23998722434043884,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2293521761894226,
+ "epoch": 7.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05877150595188141,
+ "kl": 0.00012415168203006033,
+ "learning_rate": 1.9016059735451156e-07,
+ "loss": -0.008430279791355133,
+ "num_tokens": 7535518.0,
+ "reward": 0.47240138053894043,
+ "reward_std": 0.2845028340816498,
+ "rewards/true_env_reward_fn/mean": 0.47240138053894043,
+ "rewards/true_env_reward_fn/std": 0.2845028340816498,
"step": 305,
- "step_time": 4.381737805999364
+ "step_time": 12.694503639000686
},
{
"clip_ratio/high_max": 0.0,
@@ -8556,26 +8556,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 154.0,
- "completions/max_terminated_length": 154.0,
- "completions/mean_length": 88.75,
- "completions/mean_terminated_length": 88.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.235496699810028,
- "epoch": 2.4878048780487805,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010392541298642755,
- "kl": 1.9250182049290743e-05,
- "learning_rate": 7.113759532207599e-08,
- "loss": 9.166102472590865e-07,
- "num_tokens": 1256018.0,
- "reward": 0.45198333263397217,
- "reward_std": 0.0026191486977040768,
- "rewards/true_env_reward_fn/mean": 0.45198333263397217,
- "rewards/true_env_reward_fn/std": 0.0026191489305347204,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1673714816570282,
+ "epoch": 7.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07308074831962585,
+ "kl": 0.0001147145967479446,
+ "learning_rate": 1.8683081505338465e-07,
+ "loss": -0.06511729210615158,
+ "num_tokens": 7558883.0,
+ "reward": 0.4651026129722595,
+ "reward_std": 0.19122423231601715,
+ "rewards/true_env_reward_fn/mean": 0.46510258316993713,
+ "rewards/true_env_reward_fn/std": 0.19122423231601715,
"step": 306,
- "step_time": 6.6232522029986285
+ "step_time": 15.502204728000834
},
{
"clip_ratio/high_max": 0.0,
@@ -8584,26 +8584,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.1592004895210266,
- "epoch": 2.4959349593495936,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.044789915904403e-05,
- "kl": 9.38343055167934e-06,
- "learning_rate": 6.899766394289516e-08,
- "loss": 4.709004315373022e-07,
- "num_tokens": 1259980.0,
- "reward": 0.4911326766014099,
- "reward_std": 0.01917082816362381,
- "rewards/true_env_reward_fn/mean": 0.4911326766014099,
- "rewards/true_env_reward_fn/std": 0.01917083002626896,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 63.6875,
+ "completions/mean_terminated_length": 63.6875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.304062157869339,
+ "epoch": 7.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0670589879155159,
+ "kl": 9.212431177729741e-05,
+ "learning_rate": 1.8352373262443915e-07,
+ "loss": -0.026926759630441666,
+ "num_tokens": 7578916.0,
+ "reward": 0.5716189742088318,
+ "reward_std": 0.19298586249351501,
+ "rewards/true_env_reward_fn/mean": 0.5716189742088318,
+ "rewards/true_env_reward_fn/std": 0.19298586249351501,
"step": 307,
- "step_time": 3.398790989002009
+ "step_time": 8.911180752998916
},
{
"clip_ratio/high_max": 0.0,
@@ -8612,26 +8612,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1962818503379822,
- "epoch": 2.5040650406504064,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12975044548511505,
- "kl": 9.622429843147984e-06,
- "learning_rate": 6.688737073449563e-08,
- "loss": -0.028225116431713104,
- "num_tokens": 1265125.0,
- "reward": 0.3520139753818512,
- "reward_std": 0.4090423583984375,
- "rewards/true_env_reward_fn/mean": 0.3520139753818512,
- "rewards/true_env_reward_fn/std": 0.4090423583984375,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2283632457256317,
+ "epoch": 7.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06950085610151291,
+ "kl": 9.545813054501195e-05,
+ "learning_rate": 1.80239589779473e-07,
+ "loss": 0.010137543082237244,
+ "num_tokens": 7604028.0,
+ "reward": 0.5349916219711304,
+ "reward_std": 0.2091754674911499,
+ "rewards/true_env_reward_fn/mean": 0.5349915623664856,
+ "rewards/true_env_reward_fn/std": 0.2091754525899887,
"step": 308,
- "step_time": 3.4128740100004507
+ "step_time": 11.745030509000571
},
{
"clip_ratio/high_max": 0.0,
@@ -8640,26 +8640,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 77.5,
+ "completions/mean_terminated_length": 77.5,
"completions/min_length": 37.0,
"completions/min_terminated_length": 37.0,
- "entropy": 1.1477364301681519,
- "epoch": 2.5121951219512195,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.428078242810443e-05,
- "kl": 1.2252480246388586e-05,
- "learning_rate": 6.480690465394398e-08,
- "loss": 6.147511157905683e-07,
- "num_tokens": 1269679.0,
- "reward": 0.4544333219528198,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4544333219528198,
- "rewards/true_env_reward_fn/std": 0.0,
+ "entropy": 1.3315171897411346,
+ "epoch": 7.536585365853659,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05699850618839264,
+ "kl": 6.70248282403918e-05,
+ "learning_rate": 1.7697862456752271e-07,
+ "loss": 0.06499005854129791,
+ "num_tokens": 7628884.0,
+ "reward": 0.4583873748779297,
+ "reward_std": 0.26091766357421875,
+ "rewards/true_env_reward_fn/mean": 0.4583873748779297,
+ "rewards/true_env_reward_fn/std": 0.26091763377189636,
"step": 309,
- "step_time": 3.4761773770005675
+ "step_time": 13.87453935099984
},
{
"clip_ratio/high_max": 0.0,
@@ -8668,26 +8668,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 60.625,
- "completions/mean_terminated_length": 60.625,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "entropy": 1.1964089274406433,
- "epoch": 2.5203252032520327,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00010435594595037401,
- "kl": 1.349770900560543e-05,
- "learning_rate": 6.275645198756629e-08,
- "loss": 6.71000861984794e-07,
- "num_tokens": 1272836.0,
- "reward": 0.4899469017982483,
- "reward_std": 0.24049179255962372,
- "rewards/true_env_reward_fn/mean": 0.4899469017982483,
- "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2704036831855774,
+ "epoch": 7.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09379290789365768,
+ "kl": 0.0001983325491892174,
+ "learning_rate": 1.7374107335760934e-07,
+ "loss": 0.09065254032611847,
+ "num_tokens": 7651991.0,
+ "reward": 0.5210780501365662,
+ "reward_std": 0.2745552361011505,
+ "rewards/true_env_reward_fn/mean": 0.5210780501365662,
+ "rewards/true_env_reward_fn/std": 0.2745552361011505,
"step": 310,
- "step_time": 3.522990450999714
+ "step_time": 11.07200519900016
},
{
"clip_ratio/high_max": 0.0,
@@ -8696,26 +8696,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 399.0,
- "completions/max_terminated_length": 399.0,
- "completions/mean_length": 101.125,
- "completions/mean_terminated_length": 101.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.428375780582428,
- "epoch": 2.5284552845528454,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10063836723566055,
- "kl": 2.3114703708415618e-05,
- "learning_rate": 6.073619633426675e-08,
- "loss": -0.1970823109149933,
- "num_tokens": 1279065.0,
- "reward": 0.4598180055618286,
- "reward_std": 0.23299095034599304,
- "rewards/true_env_reward_fn/mean": 0.4598180055618286,
- "rewards/true_env_reward_fn/std": 0.23299095034599304,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.60417175292969,
+ "completions/mean_terminated_length": 73.60417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2427658140659332,
+ "epoch": 7.585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06011087819933891,
+ "kl": 6.545234282384627e-05,
+ "learning_rate": 1.7052717082160344e-07,
+ "loss": -0.0115435142070055,
+ "num_tokens": 7682948.0,
+ "reward": 0.3796111047267914,
+ "reward_std": 0.2212861329317093,
+ "rewards/true_env_reward_fn/mean": 0.3796111047267914,
+ "rewards/true_env_reward_fn/std": 0.22128616273403168,
"step": 311,
- "step_time": 16.64716850199875
+ "step_time": 12.894371897999918
},
{
"clip_ratio/high_max": 0.0,
@@ -8724,26 +8724,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 62.875,
- "completions/mean_terminated_length": 62.875,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.0843100249767303,
- "epoch": 2.5365853658536586,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00015719492512289435,
- "kl": 1.4557146187144099e-05,
- "learning_rate": 5.8746318589089337e-08,
- "loss": 7.280061709025176e-07,
- "num_tokens": 1280592.0,
- "reward": 0.8541955947875977,
- "reward_std": 0.09160846471786499,
- "rewards/true_env_reward_fn/mean": 0.8541955947875977,
- "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1208404004573822,
+ "epoch": 7.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0799943059682846,
+ "kl": 0.0001426433700544294,
+ "learning_rate": 1.6733714991721738e-07,
+ "loss": -0.010216867551207542,
+ "num_tokens": 7704179.0,
+ "reward": 0.5524939298629761,
+ "reward_std": 0.26985135674476624,
+ "rewards/true_env_reward_fn/mean": 0.5524939298629761,
+ "rewards/true_env_reward_fn/std": 0.26985135674476624,
"step": 312,
- "step_time": 3.2888442910007143
+ "step_time": 8.69524126000033
},
{
"clip_ratio/high_max": 0.0,
@@ -8752,26 +8752,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 72.0,
- "completions/mean_terminated_length": 72.0,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.240959882736206,
- "epoch": 2.5447154471544717,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12602119147777557,
- "kl": 1.7604435015527997e-05,
- "learning_rate": 5.67869969270196e-08,
- "loss": 9.685754776000977e-07,
- "num_tokens": 1284772.0,
- "reward": 0.4882529377937317,
- "reward_std": 0.2615475356578827,
- "rewards/true_env_reward_fn/mean": 0.4882529377937317,
- "rewards/true_env_reward_fn/std": 0.2615475356578827,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2062768340110779,
+ "epoch": 7.634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05652271583676338,
+ "kl": 0.00010435856393087306,
+ "learning_rate": 1.6417124187111774e-07,
+ "loss": 0.033915065228939056,
+ "num_tokens": 7732974.0,
+ "reward": 0.40553492307662964,
+ "reward_std": 0.2561950087547302,
+ "rewards/true_env_reward_fn/mean": 0.40553489327430725,
+ "rewards/true_env_reward_fn/std": 0.2561950087547302,
"step": 313,
- "step_time": 4.0729800409990276
+ "step_time": 11.456125995000548
},
{
"clip_ratio/high_max": 0.0,
@@ -8780,26 +8780,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1718184351921082,
- "epoch": 2.5528455284552845,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14716969430446625,
- "kl": 1.3087485967844259e-05,
- "learning_rate": 5.4858406787030845e-08,
- "loss": 0.016231566667556763,
- "num_tokens": 1287839.0,
- "reward": 0.6037359237670898,
- "reward_std": 0.10266375541687012,
- "rewards/true_env_reward_fn/mean": 0.6037359237670898,
- "rewards/true_env_reward_fn/std": 0.10266375541687012,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 63.41666793823242,
+ "completions/mean_terminated_length": 63.41666793823242,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2864463925361633,
+ "epoch": 7.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08890142291784286,
+ "kl": 0.00014215287774277385,
+ "learning_rate": 1.6102967616216617e-07,
+ "loss": 0.04210362955927849,
+ "num_tokens": 7754650.0,
+ "reward": 0.5954334139823914,
+ "reward_std": 0.3237621784210205,
+ "rewards/true_env_reward_fn/mean": 0.5954334139823914,
+ "rewards/true_env_reward_fn/std": 0.3237621784210205,
"step": 314,
- "step_time": 3.175157601999672
+ "step_time": 11.561733381999147
},
{
"clip_ratio/high_max": 0.0,
@@ -8808,26 +8808,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 52.0,
- "completions/max_terminated_length": 52.0,
- "completions/mean_length": 40.5,
- "completions/mean_terminated_length": 40.5,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.3117390871047974,
- "epoch": 2.5609756097560976,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2794886827468872,
- "kl": 4.65317443740787e-05,
- "learning_rate": 5.29607208563756e-08,
- "loss": -0.09303665161132812,
- "num_tokens": 1291515.0,
- "reward": 0.5445280075073242,
- "reward_std": 0.19415061175823212,
- "rewards/true_env_reward_fn/mean": 0.5445280075073242,
- "rewards/true_env_reward_fn/std": 0.19415059685707092,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 69.22917175292969,
+ "completions/mean_terminated_length": 69.22917175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2406023740768433,
+ "epoch": 7.682926829268292,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05483023077249527,
+ "kl": 3.3986466860369546e-05,
+ "learning_rate": 1.5791268050478483e-07,
+ "loss": -0.03052404522895813,
+ "num_tokens": 7777389.0,
+ "reward": 0.5392192602157593,
+ "reward_std": 0.21974749863147736,
+ "rewards/true_env_reward_fn/mean": 0.5392192006111145,
+ "rewards/true_env_reward_fn/std": 0.21974751353263855,
"step": 315,
- "step_time": 2.636586960999921
+ "step_time": 13.48241268899983
},
{
"clip_ratio/high_max": 0.0,
@@ -8836,26 +8836,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 13.0,
- "completions/min_terminated_length": 13.0,
- "entropy": 1.15413236618042,
- "epoch": 2.569105691056911,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22290591895580292,
- "kl": 6.188569841469871e-05,
- "learning_rate": 5.10941090551225e-08,
- "loss": -0.013920806348323822,
- "num_tokens": 1296399.0,
- "reward": 0.3418610095977783,
- "reward_std": 0.29834023118019104,
- "rewards/true_env_reward_fn/mean": 0.3418610095977783,
- "rewards/true_env_reward_fn/std": 0.2983402609825134,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2167797982692719,
+ "epoch": 7.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08746550232172012,
+ "kl": 0.00011399560207792092,
+ "learning_rate": 1.5482048083245114e-07,
+ "loss": 0.014903642237186432,
+ "num_tokens": 7807828.0,
+ "reward": 0.29947829246520996,
+ "reward_std": 0.2547810673713684,
+ "rewards/true_env_reward_fn/mean": 0.29947829246520996,
+ "rewards/true_env_reward_fn/std": 0.2547810673713684,
"step": 316,
- "step_time": 3.7106533750011295
+ "step_time": 10.080044547000398
},
{
"clip_ratio/high_max": 0.0,
@@ -8864,26 +8864,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 265.0,
- "completions/max_terminated_length": 265.0,
- "completions/mean_length": 102.875,
- "completions/mean_terminated_length": 102.875,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.2713149785995483,
- "epoch": 2.5772357723577235,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11319029331207275,
- "kl": 1.3128932096151402e-05,
- "learning_rate": 4.9258738520942025e-08,
- "loss": 0.04848237335681915,
- "num_tokens": 1301626.0,
- "reward": 0.4802166819572449,
- "reward_std": 0.05550921708345413,
- "rewards/true_env_reward_fn/mean": 0.4802166819572449,
- "rewards/true_env_reward_fn/std": 0.05550922453403473,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 65.3125,
+ "completions/mean_terminated_length": 65.3125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2823624312877655,
+ "epoch": 7.7317073170731705,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07846319675445557,
+ "kl": 0.00012852110194216948,
+ "learning_rate": 1.517533012813217e-07,
+ "loss": 0.05300665646791458,
+ "num_tokens": 7834227.0,
+ "reward": 0.4816446304321289,
+ "reward_std": 0.3394080102443695,
+ "rewards/true_env_reward_fn/mean": 0.4816446304321289,
+ "rewards/true_env_reward_fn/std": 0.3394079804420471,
"step": 317,
- "step_time": 11.140286670997739
+ "step_time": 12.856840839001052
},
{
"clip_ratio/high_max": 0.0,
@@ -8892,26 +8892,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 64.0,
- "completions/max_terminated_length": 64.0,
- "completions/mean_length": 45.625,
- "completions/mean_terminated_length": 45.625,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 0.9857950508594513,
- "epoch": 2.5853658536585367,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011810586147475988,
- "kl": 1.2108953342249151e-05,
- "learning_rate": 4.745477359414045e-08,
- "loss": 6.04832052886195e-07,
- "num_tokens": 1303619.0,
- "reward": 0.6204532384872437,
- "reward_std": 0.10097470134496689,
- "rewards/true_env_reward_fn/mean": 0.6204532384872437,
- "rewards/true_env_reward_fn/std": 0.10097470879554749,
+ "completions/max_length": 264.0,
+ "completions/max_terminated_length": 264.0,
+ "completions/mean_length": 83.47917175292969,
+ "completions/mean_terminated_length": 83.47917175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3714390099048615,
+ "epoch": 7.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0722746029496193,
+ "kl": 0.00011485655522847082,
+ "learning_rate": 1.4871136417398405e-07,
+ "loss": -0.009422918781638145,
+ "num_tokens": 7860362.0,
+ "reward": 0.44833892583847046,
+ "reward_std": 0.24655339121818542,
+ "rewards/true_env_reward_fn/mean": 0.44833889603614807,
+ "rewards/true_env_reward_fn/std": 0.24655337631702423,
"step": 318,
- "step_time": 2.977002043000539
+ "step_time": 18.09142264499951
},
{
"clip_ratio/high_max": 0.0,
@@ -8920,26 +8920,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2560631036758423,
- "epoch": 2.59349593495935,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2680339813232422,
+ "epoch": 7.780487804878049,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.24155104160308838,
- "kl": 3.4935148505610414e-05,
- "learning_rate": 4.5682375802945295e-08,
- "loss": 0.1817030906677246,
- "num_tokens": 1310682.0,
- "reward": 0.055160000920295715,
- "reward_std": 0.13755998015403748,
- "rewards/true_env_reward_fn/mean": 0.055160000920295715,
- "rewards/true_env_reward_fn/std": 0.13755998015403748,
+ "grad_norm": 0.09518000483512878,
+ "kl": 0.00010953140827041352,
+ "learning_rate": 1.4569489000334433e-07,
+ "loss": -0.09500816464424133,
+ "num_tokens": 7882799.0,
+ "reward": 0.4905685782432556,
+ "reward_std": 0.261008620262146,
+ "rewards/true_env_reward_fn/mean": 0.4905685484409332,
+ "rewards/true_env_reward_fn/std": 0.261008620262146,
"step": 319,
- "step_time": 4.712957282001298
+ "step_time": 17.2091521250004
},
{
"clip_ratio/high_max": 0.0,
@@ -8948,26 +8948,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 102.0,
- "completions/max_terminated_length": 102.0,
- "completions/mean_length": 69.75,
- "completions/mean_terminated_length": 69.75,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2313570380210876,
- "epoch": 2.6016260162601625,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011636545968940482,
- "kl": 1.566714945511194e-05,
- "learning_rate": 4.394170384904137e-08,
- "loss": 7.867492968216538e-07,
- "num_tokens": 1315436.0,
- "reward": 0.4662666618824005,
- "reward_std": 0.007412042934447527,
- "rewards/true_env_reward_fn/mean": 0.4662666618824005,
- "rewards/true_env_reward_fn/std": 0.007412043400108814,
+ "completions/max_length": 257.0,
+ "completions/max_terminated_length": 257.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2107920348644257,
+ "epoch": 7.804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07063736766576767,
+ "kl": 3.921870529666194e-05,
+ "learning_rate": 1.4270409741664268e-07,
+ "loss": -0.011502111330628395,
+ "num_tokens": 7901717.0,
+ "reward": 0.566825807094574,
+ "reward_std": 0.18909600377082825,
+ "rewards/true_env_reward_fn/mean": 0.566825807094574,
+ "rewards/true_env_reward_fn/std": 0.18909598886966705,
"step": 320,
- "step_time": 4.536037327998201
+ "step_time": 15.649325063000106
},
{
"clip_ratio/high_max": 0.0,
@@ -8976,26 +8976,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 49.625,
- "completions/mean_terminated_length": 49.625,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.0671695470809937,
- "epoch": 2.6097560975609757,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00013134608161635697,
- "kl": 1.7747691344993655e-05,
- "learning_rate": 4.223291359336074e-08,
- "loss": 8.864202527547604e-07,
- "num_tokens": 1318837.0,
- "reward": 0.6651140451431274,
- "reward_std": 0.20516473054885864,
- "rewards/true_env_reward_fn/mean": 0.6651140451431274,
- "rewards/true_env_reward_fn/std": 0.20516474545001984,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 66.66667175292969,
+ "completions/mean_terminated_length": 66.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2565636038780212,
+ "epoch": 7.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06063119322061539,
+ "kl": 4.300068394513801e-05,
+ "learning_rate": 1.3973920319960652e-07,
+ "loss": 0.001966973766684532,
+ "num_tokens": 7919013.0,
+ "reward": 0.6115278005599976,
+ "reward_std": 0.17882205545902252,
+ "rewards/true_env_reward_fn/mean": 0.6115277409553528,
+ "rewards/true_env_reward_fn/std": 0.17882204055786133,
"step": 321,
- "step_time": 2.9819667609990574
+ "step_time": 7.359487544999865
},
{
"clip_ratio/high_max": 0.0,
@@ -9004,26 +9004,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 178.0,
- "completions/max_terminated_length": 178.0,
- "completions/mean_length": 85.25,
- "completions/mean_terminated_length": 85.25,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3910706639289856,
- "epoch": 2.617886178861789,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1727662831544876,
- "kl": 1.9147763850924093e-05,
- "learning_rate": 4.05561580421272e-08,
- "loss": 0.07752666622400284,
- "num_tokens": 1324515.0,
- "reward": 0.43344998359680176,
- "reward_std": 0.039401449263095856,
- "rewards/true_env_reward_fn/mean": 0.43344998359680176,
- "rewards/true_env_reward_fn/std": 0.039401449263095856,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 57.083335876464844,
+ "completions/mean_terminated_length": 57.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2200327813625336,
+ "epoch": 7.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07409472018480301,
+ "kl": 0.00011575021017051768,
+ "learning_rate": 1.368004222607355e-07,
+ "loss": -0.047135986387729645,
+ "num_tokens": 7941137.0,
+ "reward": 0.5057475566864014,
+ "reward_std": 0.3366250991821289,
+ "rewards/true_env_reward_fn/mean": 0.5057475566864014,
+ "rewards/true_env_reward_fn/std": 0.3366251289844513,
"step": 322,
- "step_time": 7.645187717998851
+ "step_time": 10.691665401999671
},
{
"clip_ratio/high_max": 0.0,
@@ -9032,26 +9032,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 92.0,
- "completions/max_terminated_length": 92.0,
- "completions/mean_length": 65.125,
- "completions/mean_terminated_length": 65.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.1752581596374512,
- "epoch": 2.6260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14819224178791046,
- "kl": 1.1951405213039834e-05,
- "learning_rate": 3.891158733315505e-08,
- "loss": 0.0940035805106163,
- "num_tokens": 1327268.0,
- "reward": 0.6361198425292969,
- "reward_std": 0.13543039560317993,
- "rewards/true_env_reward_fn/mean": 0.6361198425292969,
- "rewards/true_env_reward_fn/std": 0.13543038070201874,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 57.520835876464844,
+ "completions/mean_terminated_length": 57.520835876464844,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.3097383677959442,
+ "epoch": 7.878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0950520932674408,
+ "kl": 0.00014910039135429543,
+ "learning_rate": 1.338879676157249e-07,
+ "loss": -0.018300604075193405,
+ "num_tokens": 7967298.0,
+ "reward": 0.39754772186279297,
+ "reward_std": 0.227029949426651,
+ "rewards/true_env_reward_fn/mean": 0.39754772186279297,
+ "rewards/true_env_reward_fn/std": 0.2270299643278122,
"step": 323,
- "step_time": 4.102753150000353
+ "step_time": 9.646710404997975
},
{
"clip_ratio/high_max": 0.0,
@@ -9060,26 +9060,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 69.75,
- "completions/mean_terminated_length": 69.75,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.4734079241752625,
- "epoch": 2.6341463414634148,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21136029064655304,
- "kl": 4.428037027537357e-05,
- "learning_rate": 3.7299348722406965e-08,
- "loss": -0.13074398040771484,
- "num_tokens": 1331614.0,
- "reward": 0.3051303029060364,
- "reward_std": 0.20326493680477142,
- "rewards/true_env_reward_fn/mean": 0.3051303029060364,
- "rewards/true_env_reward_fn/std": 0.20326495170593262,
+ "completions/max_length": 165.0,
+ "completions/max_terminated_length": 165.0,
+ "completions/mean_length": 63.04166793823242,
+ "completions/mean_terminated_length": 63.04166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3998334109783173,
+ "epoch": 7.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08187483251094818,
+ "kl": 0.0001343226194876479,
+ "learning_rate": 1.310020503720254e-07,
+ "loss": 0.10223554074764252,
+ "num_tokens": 7989452.0,
+ "reward": 0.4514659643173218,
+ "reward_std": 0.3259531557559967,
+ "rewards/true_env_reward_fn/mean": 0.4514659643173218,
+ "rewards/true_env_reward_fn/std": 0.3259531557559967,
"step": 324,
- "step_time": 5.413495869999679
+ "step_time": 13.284335969000495
},
{
"clip_ratio/high_max": 0.0,
@@ -9088,26 +9088,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 179.0,
- "completions/max_terminated_length": 179.0,
- "completions/mean_length": 78.625,
- "completions/mean_terminated_length": 78.625,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.5631850361824036,
- "epoch": 2.642276422764228,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18741382658481598,
- "kl": 3.721813300217036e-05,
- "learning_rate": 3.57195865708074e-08,
- "loss": 0.20410414040088654,
- "num_tokens": 1334991.0,
- "reward": 0.4402502775192261,
- "reward_std": 0.27313899993896484,
- "rewards/true_env_reward_fn/mean": 0.4402502775192261,
- "rewards/true_env_reward_fn/std": 0.27313899993896484,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 78.77083587646484,
+ "completions/mean_terminated_length": 78.77083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3267191052436829,
+ "epoch": 7.926829268292683,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06648680567741394,
+ "kl": 5.966442222415935e-05,
+ "learning_rate": 1.281428797135402e-07,
+ "loss": -0.046651843935251236,
+ "num_tokens": 8011065.0,
+ "reward": 0.6008384227752686,
+ "reward_std": 0.19184507429599762,
+ "rewards/true_env_reward_fn/mean": 0.6008384227752686,
+ "rewards/true_env_reward_fn/std": 0.19184507429599762,
"step": 325,
- "step_time": 7.439067015000546
+ "step_time": 18.06017050799983
},
{
"clip_ratio/high_max": 0.0,
@@ -9116,26 +9116,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.2137504816055298,
- "epoch": 2.6504065040650406,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1392536610364914,
- "kl": 2.5514624212519266e-05,
- "learning_rate": 3.417244233131709e-08,
- "loss": -0.07728786766529083,
- "num_tokens": 1340886.0,
- "reward": 0.3032499849796295,
- "reward_std": 0.24293993413448334,
- "rewards/true_env_reward_fn/mean": 0.3032499849796295,
- "rewards/true_env_reward_fn/std": 0.24293994903564453,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 61.54166793823242,
+ "completions/mean_terminated_length": 61.54166793823242,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.4001933634281158,
+ "epoch": 7.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10428700596094131,
+ "kl": 0.0001954599174496252,
+ "learning_rate": 1.253106628854635e-07,
+ "loss": -0.08898806571960449,
+ "num_tokens": 8042035.0,
+ "reward": 0.21636998653411865,
+ "reward_std": 0.26617059111595154,
+ "rewards/true_env_reward_fn/mean": 0.21636998653411865,
+ "rewards/true_env_reward_fn/std": 0.26617059111595154,
"step": 326,
- "step_time": 3.6837621579998086
+ "step_time": 9.872497149000083
},
{
"clip_ratio/high_max": 0.0,
@@ -9144,26 +9144,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 45.125,
- "completions/mean_terminated_length": 45.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1947113871574402,
- "epoch": 2.658536585365854,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23312504589557648,
- "kl": 6.829304766142741e-05,
- "learning_rate": 3.265805453626713e-08,
- "loss": 0.08558641374111176,
- "num_tokens": 1344823.0,
- "reward": 0.26869305968284607,
- "reward_std": 0.30387282371520996,
- "rewards/true_env_reward_fn/mean": 0.26869305968284607,
- "rewards/true_env_reward_fn/std": 0.30387285351753235,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 69.91667175292969,
+ "completions/mean_terminated_length": 69.91667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.198440283536911,
+ "epoch": 7.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06512034684419632,
+ "kl": 9.893041169561911e-05,
+ "learning_rate": 1.2250560517925745e-07,
+ "loss": -0.004943478852510452,
+ "num_tokens": 8066991.0,
+ "reward": 0.5199041962623596,
+ "reward_std": 0.2916473150253296,
+ "rewards/true_env_reward_fn/mean": 0.5199041962623596,
+ "rewards/true_env_reward_fn/std": 0.2916473150253296,
"step": 327,
- "step_time": 3.26592919200084
+ "step_time": 13.27990607999891
},
{
"clip_ratio/high_max": 0.0,
@@ -9172,26 +9172,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 70.125,
- "completions/mean_terminated_length": 70.125,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.1701073944568634,
- "epoch": 2.6666666666666665,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12238985300064087,
- "kl": 1.6267454157059547e-05,
- "learning_rate": 3.117655878495458e-08,
- "loss": -0.024053269997239113,
- "num_tokens": 1348012.0,
- "reward": 0.5990839004516602,
- "reward_std": 0.21510076522827148,
- "rewards/true_env_reward_fn/mean": 0.5990839004516602,
- "rewards/true_env_reward_fn/std": 0.21510076522827148,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.3655813038349152,
+ "epoch": 8.0,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08734066039323807,
+ "kl": 0.00011680843272188213,
+ "learning_rate": 1.197279099177731e-07,
+ "loss": -0.027742527425289154,
+ "num_tokens": 8095161.0,
+ "reward": 0.4403582811355591,
+ "reward_std": 0.20052418112754822,
+ "rewards/true_env_reward_fn/mean": 0.4403582811355591,
+ "rewards/true_env_reward_fn/std": 0.20052418112754822,
"step": 328,
- "step_time": 4.166060536001169
+ "step_time": 9.091917235000437
},
{
"clip_ratio/high_max": 0.0,
@@ -9200,26 +9200,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 49.75,
- "completions/mean_terminated_length": 49.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.3588789701461792,
- "epoch": 2.6747967479674797,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21684816479682922,
- "kl": 0.00012067620264133438,
- "learning_rate": 2.9728087731500884e-08,
- "loss": 0.017311420291662216,
- "num_tokens": 1352082.0,
- "reward": 0.15680472552776337,
- "reward_std": 0.3331984877586365,
- "rewards/true_env_reward_fn/mean": 0.15680472552776337,
- "rewards/true_env_reward_fn/std": 0.3331984877586365,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.645835876464844,
+ "completions/mean_terminated_length": 63.645835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1942758560180664,
+ "epoch": 8.024390243902438,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07103214412927628,
+ "kl": 9.241796033165883e-05,
+ "learning_rate": 1.1697777844051104e-07,
+ "loss": -0.024517521262168884,
+ "num_tokens": 8117752.0,
+ "reward": 0.4961467981338501,
+ "reward_std": 0.2681204676628113,
+ "rewards/true_env_reward_fn/mean": 0.4961467981338501,
+ "rewards/true_env_reward_fn/std": 0.2681204676628113,
"step": 329,
- "step_time": 2.947034541000903
+ "step_time": 10.042522196999926
},
{
"clip_ratio/high_max": 0.0,
@@ -9228,26 +9228,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 57.125,
- "completions/mean_terminated_length": 57.125,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "entropy": 1.2207121849060059,
- "epoch": 2.682926829268293,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14501330256462097,
- "kl": 3.47198892995948e-05,
- "learning_rate": 2.831277107297394e-08,
- "loss": -0.009820610284805298,
- "num_tokens": 1356867.0,
- "reward": 0.508467972278595,
- "reward_std": 0.10515833646059036,
- "rewards/true_env_reward_fn/mean": 0.508467972278595,
- "rewards/true_env_reward_fn/std": 0.10515833646059036,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 71.58333587646484,
+ "completions/mean_terminated_length": 71.58333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.209133356809616,
+ "epoch": 8.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06019354239106178,
+ "kl": 2.6403215088066645e-05,
+ "learning_rate": 1.142554100890285e-07,
+ "loss": -0.03160044550895691,
+ "num_tokens": 8140164.0,
+ "reward": 0.5655641555786133,
+ "reward_std": 0.1564219743013382,
+ "rewards/true_env_reward_fn/mean": 0.5655641555786133,
+ "rewards/true_env_reward_fn/std": 0.1564219743013382,
"step": 330,
- "step_time": 3.278030399000272
+ "step_time": 8.212663780001094
},
{
"clip_ratio/high_max": 0.0,
@@ -9256,26 +9256,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 121.0,
- "completions/max_terminated_length": 121.0,
- "completions/mean_length": 71.5,
- "completions/mean_terminated_length": 71.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2053698897361755,
- "epoch": 2.6910569105691056,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.1709476113319397,
- "kl": 1.9107792468275875e-05,
- "learning_rate": 2.693073553777494e-08,
- "loss": 0.04886455833911896,
- "num_tokens": 1361747.0,
- "reward": 0.3092518746852875,
- "reward_std": 0.40765801072120667,
- "rewards/true_env_reward_fn/mean": 0.3092518746852875,
- "rewards/true_env_reward_fn/std": 0.40765801072120667,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.229485958814621,
+ "epoch": 8.073170731707316,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07769559323787689,
+ "kl": 0.0001446352634957293,
+ "learning_rate": 1.115610021924902e-07,
+ "loss": -0.01835213601589203,
+ "num_tokens": 8177099.0,
+ "reward": 0.1968289315700531,
+ "reward_std": 0.3009001910686493,
+ "rewards/true_env_reward_fn/mean": 0.1968289166688919,
+ "rewards/true_env_reward_fn/std": 0.3009001612663269,
"step": 331,
- "step_time": 5.705391269999382
+ "step_time": 16.508294159000798
},
{
"clip_ratio/high_max": 0.0,
@@ -9284,26 +9284,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 70.625,
- "completions/mean_terminated_length": 70.625,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.3019116520881653,
- "epoch": 2.6991869918699187,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13118988275527954,
- "kl": 6.723498518113047e-05,
- "learning_rate": 2.5582104874290888e-08,
- "loss": 0.04069610685110092,
- "num_tokens": 1366792.0,
- "reward": 0.39014729857444763,
- "reward_std": 0.4048542380332947,
- "rewards/true_env_reward_fn/mean": 0.39014729857444763,
- "rewards/true_env_reward_fn/std": 0.40485426783561707,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 55.72916793823242,
+ "completions/mean_terminated_length": 55.72916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.249097228050232,
+ "epoch": 8.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09750289469957352,
+ "kl": 0.00022061014351493213,
+ "learning_rate": 1.0889475005336446e-07,
+ "loss": 0.012823417782783508,
+ "num_tokens": 8202310.0,
+ "reward": 0.4960649907588959,
+ "reward_std": 0.28122010827064514,
+ "rewards/true_env_reward_fn/mean": 0.4960649907588959,
+ "rewards/true_env_reward_fn/std": 0.28122007846832275,
"step": 332,
- "step_time": 4.941442722001739
+ "step_time": 9.309349606999604
},
{
"clip_ratio/high_max": 0.0,
@@ -9311,27 +9311,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.125,
+ "completions/clipped_ratio": 0.02083333395421505,
"completions/max_length": 512.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 122.25,
- "completions/mean_terminated_length": 66.5714340209961,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 0.8574695587158203,
- "epoch": 2.7073170731707314,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.08299780637025833,
- "kl": 8.204095138353296e-06,
- "learning_rate": 2.4266999839814484e-08,
- "loss": -0.5517755150794983,
- "num_tokens": 1373066.0,
- "reward": 0.19574999809265137,
- "reward_std": 0.30308666825294495,
- "rewards/true_env_reward_fn/mean": 0.19574999809265137,
- "rewards/true_env_reward_fn/std": 0.30308666825294495,
+ "completions/max_terminated_length": 326.0,
+ "completions/mean_length": 82.35417175292969,
+ "completions/mean_terminated_length": 73.21276092529297,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3090683817863464,
+ "epoch": 8.121951219512194,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09015674889087677,
+ "kl": 0.00010220101285085548,
+ "learning_rate": 1.0625684693326725e-07,
+ "loss": 0.23226313292980194,
+ "num_tokens": 8227663.0,
+ "reward": 0.4449918270111084,
+ "reward_std": 0.24675849080085754,
+ "rewards/true_env_reward_fn/mean": 0.4449918270111084,
+ "rewards/true_env_reward_fn/std": 0.24675849080085754,
"step": 333,
- "step_time": 20.7688721859995
+ "step_time": 29.62071167800059
},
{
"clip_ratio/high_max": 0.0,
@@ -9340,26 +9340,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 49.625,
- "completions/mean_terminated_length": 49.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.191849946975708,
- "epoch": 2.7154471544715446,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.0001034519009408541,
- "kl": 1.4263560842664447e-05,
- "learning_rate": 2.2985538189730946e-08,
- "loss": 7.244366315717343e-07,
- "num_tokens": 1375815.0,
- "reward": 0.6581807136535645,
- "reward_std": 0.21257677674293518,
- "rewards/true_env_reward_fn/mean": 0.6581807136535645,
- "rewards/true_env_reward_fn/std": 0.21257677674293518,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 67.52083587646484,
+ "completions/mean_terminated_length": 67.52083587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2623873353004456,
+ "epoch": 8.146341463414634,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09357088804244995,
+ "kl": 0.00020532654161797836,
+ "learning_rate": 1.0364748403895368e-07,
+ "loss": 0.015600509941577911,
+ "num_tokens": 8251736.0,
+ "reward": 0.49391698837280273,
+ "reward_std": 0.313683420419693,
+ "rewards/true_env_reward_fn/mean": 0.49391698837280273,
+ "rewards/true_env_reward_fn/std": 0.3136834502220154,
"step": 334,
- "step_time": 3.5656982629971026
+ "step_time": 14.77577510500032
},
{
"clip_ratio/high_max": 0.0,
@@ -9368,26 +9368,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 59.75,
- "completions/mean_terminated_length": 59.75,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.2259244322776794,
- "epoch": 2.7235772357723578,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12885747849941254,
- "kl": 4.455958514881786e-05,
- "learning_rate": 2.1737834666974276e-08,
- "loss": -0.06481382250785828,
- "num_tokens": 1379933.0,
- "reward": 0.31903696060180664,
- "reward_std": 0.1877410113811493,
- "rewards/true_env_reward_fn/mean": 0.31903696060180664,
- "rewards/true_env_reward_fn/std": 0.18774102628231049,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 89.25,
+ "completions/mean_terminated_length": 89.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.3438844978809357,
+ "epoch": 8.170731707317072,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07505634427070618,
+ "kl": 8.66996942932019e-05,
+ "learning_rate": 1.0106685050845837e-07,
+ "loss": -0.006955280434340239,
+ "num_tokens": 8283596.0,
+ "reward": 0.37073618173599243,
+ "reward_std": 0.22015252709388733,
+ "rewards/true_env_reward_fn/mean": 0.37073615193367004,
+ "rewards/true_env_reward_fn/std": 0.22015254199504852,
"step": 335,
- "step_time": 3.712153099000716
+ "step_time": 14.388228956000603
},
{
"clip_ratio/high_max": 0.0,
@@ -9396,26 +9396,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 45.0,
- "completions/mean_terminated_length": 45.0,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "entropy": 1.2184038758277893,
- "epoch": 2.7317073170731705,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.258829265832901,
- "kl": 6.149329965410288e-05,
- "learning_rate": 2.0524000991753465e-08,
- "loss": -0.04611395671963692,
- "num_tokens": 1383205.0,
- "reward": 0.41517797112464905,
- "reward_std": 0.33201757073402405,
- "rewards/true_env_reward_fn/mean": 0.41517797112464905,
- "rewards/true_env_reward_fn/std": 0.33201757073402405,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3619366586208344,
+ "epoch": 8.195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07524604350328445,
+ "kl": 5.828110352013027e-05,
+ "learning_rate": 9.851513339738626e-08,
+ "loss": 0.02218097448348999,
+ "num_tokens": 8304798.0,
+ "reward": 0.6131325960159302,
+ "reward_std": 0.12095426768064499,
+ "rewards/true_env_reward_fn/mean": 0.6131325364112854,
+ "rewards/true_env_reward_fn/std": 0.12095426768064499,
"step": 336,
- "step_time": 3.02953305399933
+ "step_time": 12.145640587000344
},
{
"clip_ratio/high_max": 0.0,
@@ -9423,27 +9423,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completions/clipped_ratio": 0.0,
- "completions/max_length": 206.0,
- "completions/max_terminated_length": 206.0,
- "completions/mean_length": 87.875,
- "completions/mean_terminated_length": 87.875,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.2387716174125671,
- "epoch": 2.7398373983739837,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.16021011769771576,
- "kl": 1.6921320366236614e-05,
- "learning_rate": 1.9344145851548243e-08,
- "loss": -0.14579075574874878,
- "num_tokens": 1389136.0,
- "reward": 0.4916413426399231,
- "reward_std": 0.22573381662368774,
- "rewards/true_env_reward_fn/mean": 0.4916413426399231,
- "rewards/true_env_reward_fn/std": 0.22573381662368774,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 76.77083587646484,
+ "completions/mean_terminated_length": 67.51063537597656,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.6089566349983215,
+ "epoch": 8.21951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07641154527664185,
+ "kl": 0.00011577324767131358,
+ "learning_rate": 9.599251766535343e-08,
+ "loss": -0.047279007732868195,
+ "num_tokens": 8331715.0,
+ "reward": 0.37190374732017517,
+ "reward_std": 0.2730160653591156,
+ "rewards/true_env_reward_fn/mean": 0.37190374732017517,
+ "rewards/true_env_reward_fn/std": 0.2730160653591156,
"step": 337,
- "step_time": 9.208789625999998
+ "step_time": 33.08387189299992
},
{
"clip_ratio/high_max": 0.0,
@@ -9452,26 +9452,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 50.75,
- "completions/mean_terminated_length": 50.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.2942972779273987,
- "epoch": 2.747967479674797,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 60.79166793823242,
+ "completions/mean_terminated_length": 60.79166793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2664433717727661,
+ "epoch": 8.24390243902439,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.2804953157901764,
- "kl": 6.65090201437124e-05,
- "learning_rate": 1.8198374891377966e-08,
- "loss": 0.14034238457679749,
- "num_tokens": 1392442.0,
- "reward": 0.458748459815979,
- "reward_std": 0.24402853846549988,
- "rewards/true_env_reward_fn/mean": 0.458748459815979,
- "rewards/true_env_reward_fn/std": 0.24402853846549988,
+ "grad_norm": 0.10079872608184814,
+ "kl": 0.00016995913574646693,
+ "learning_rate": 9.349918616258113e-08,
+ "loss": 0.04466569796204567,
+ "num_tokens": 8357801.0,
+ "reward": 0.4597291648387909,
+ "reward_std": 0.2547009587287903,
+ "rewards/true_env_reward_fn/mean": 0.4597291648387909,
+ "rewards/true_env_reward_fn/std": 0.2547009289264679,
"step": 338,
- "step_time": 3.383874344001015
+ "step_time": 12.540567380000539
},
{
"clip_ratio/high_max": 0.0,
@@ -9480,26 +9480,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 99.0,
- "completions/max_terminated_length": 99.0,
- "completions/mean_length": 75.625,
- "completions/mean_terminated_length": 75.625,
- "completions/min_length": 60.0,
- "completions/min_terminated_length": 60.0,
- "entropy": 1.433261752128601,
- "epoch": 2.7560975609756095,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11875848472118378,
- "kl": 2.3840807443775702e-05,
- "learning_rate": 1.7086790704341402e-08,
- "loss": -0.015734028071165085,
- "num_tokens": 1396235.0,
- "reward": 0.3571140468120575,
- "reward_std": 0.10167346149682999,
- "rewards/true_env_reward_fn/mean": 0.3571140468120575,
- "rewards/true_env_reward_fn/std": 0.10167345404624939,
+ "completions/max_length": 278.0,
+ "completions/max_terminated_length": 278.0,
+ "completions/mean_length": 72.95833587646484,
+ "completions/mean_terminated_length": 72.95833587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.20101597905159,
+ "epoch": 8.268292682926829,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0724605992436409,
+ "kl": 8.693387826497201e-05,
+ "learning_rate": 9.103531961664118e-08,
+ "loss": -0.04201607406139374,
+ "num_tokens": 8391047.0,
+ "reward": 0.272682249546051,
+ "reward_std": 0.22370731830596924,
+ "rewards/true_env_reward_fn/mean": 0.27268221974372864,
+ "rewards/true_env_reward_fn/std": 0.22370733320713043,
"step": 339,
- "step_time": 4.3859739109993825
+ "step_time": 21.499952529999973
},
{
"clip_ratio/high_max": 0.0,
@@ -9508,26 +9508,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 186.0,
- "completions/max_terminated_length": 186.0,
- "completions/mean_length": 97.75,
- "completions/mean_terminated_length": 97.75,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.5857495069503784,
- "epoch": 2.7642276422764227,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 6.698282959405333e-05,
- "kl": 1.4666956758446759e-05,
- "learning_rate": 1.600949282243089e-08,
- "loss": 7.313840342249023e-07,
- "num_tokens": 1401073.0,
- "reward": 0.3621603846549988,
- "reward_std": 0.10388225317001343,
- "rewards/true_env_reward_fn/mean": 0.3621603846549988,
- "rewards/true_env_reward_fn/std": 0.10388225317001343,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 66.27083587646484,
+ "completions/mean_terminated_length": 66.27083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.240132600069046,
+ "epoch": 8.292682926829269,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09206120669841766,
+ "kl": 9.196988685289398e-05,
+ "learning_rate": 8.860109661935672e-08,
+ "loss": 0.013995083048939705,
+ "num_tokens": 8412060.0,
+ "reward": 0.5260962843894958,
+ "reward_std": 0.16053001582622528,
+ "rewards/true_env_reward_fn/mean": 0.5260962843894958,
+ "rewards/true_env_reward_fn/std": 0.16053001582622528,
"step": 340,
- "step_time": 7.751754675999109
+ "step_time": 11.287615471999743
},
{
"clip_ratio/high_max": 0.0,
@@ -9536,26 +9536,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 54.0,
- "completions/mean_terminated_length": 54.0,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.0097725987434387,
- "epoch": 2.772357723577236,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011211638775421306,
- "kl": 1.3342013517103624e-05,
- "learning_rate": 1.496657770762013e-08,
- "loss": 6.635225417994661e-07,
- "num_tokens": 1403921.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 187.0,
+ "completions/max_terminated_length": 187.0,
+ "completions/mean_length": 69.85417175292969,
+ "completions/mean_terminated_length": 69.85417175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2434665858745575,
+ "epoch": 8.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06606078892946243,
+ "kl": 0.00013524454698199406,
+ "learning_rate": 8.619669361385662e-08,
+ "loss": 0.04330434650182724,
+ "num_tokens": 8435189.0,
+ "reward": 0.5729736089706421,
+ "reward_std": 0.28772154450416565,
+ "rewards/true_env_reward_fn/mean": 0.5729735493659973,
+ "rewards/true_env_reward_fn/std": 0.28772154450416565,
"step": 341,
- "step_time": 3.4684136910000234
+ "step_time": 14.554103153000142
},
{
"clip_ratio/high_max": 0.0,
@@ -9564,26 +9564,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.1732905507087708,
- "epoch": 2.7804878048780486,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14162735641002655,
- "kl": 4.145995080762077e-05,
- "learning_rate": 1.3958138743226688e-08,
- "loss": 0.007000558078289032,
- "num_tokens": 1407681.0,
- "reward": 0.5386021733283997,
- "reward_std": 0.31964072585105896,
- "rewards/true_env_reward_fn/mean": 0.5386021733283997,
- "rewards/true_env_reward_fn/std": 0.31964072585105896,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 87.83333587646484,
+ "completions/mean_terminated_length": 87.83333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.193087100982666,
+ "epoch": 8.341463414634147,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06404980272054672,
+ "kl": 7.958761125337332e-05,
+ "learning_rate": 8.382228488178638e-08,
+ "loss": -0.02654789574444294,
+ "num_tokens": 8461485.0,
+ "reward": 0.4391756057739258,
+ "reward_std": 0.3396260142326355,
+ "rewards/true_env_reward_fn/mean": 0.4391756057739258,
+ "rewards/true_env_reward_fn/std": 0.3396260142326355,
"step": 342,
- "step_time": 3.5975158429992007
+ "step_time": 16.93046250899988
},
{
"clip_ratio/high_max": 0.0,
@@ -9592,26 +9592,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.0845041275024414,
- "epoch": 2.7886178861788617,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.0979820266366005,
- "kl": 1.916076598718064e-05,
- "learning_rate": 1.29842662255506e-08,
- "loss": -0.049884289503097534,
- "num_tokens": 1413182.0,
- "reward": 0.316629558801651,
- "reward_std": 0.16504530608654022,
- "rewards/true_env_reward_fn/mean": 0.316629558801651,
- "rewards/true_env_reward_fn/std": 0.16504532098770142,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2198111712932587,
+ "epoch": 8.365853658536585,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06700363755226135,
+ "kl": 3.631071058407542e-05,
+ "learning_rate": 8.14780425306758e-08,
+ "loss": -0.01248769462108612,
+ "num_tokens": 8477128.0,
+ "reward": 0.6880009174346924,
+ "reward_std": 0.16646623611450195,
+ "rewards/true_env_reward_fn/mean": 0.6880009174346924,
+ "rewards/true_env_reward_fn/std": 0.16646623611450195,
"step": 343,
- "step_time": 4.6961259659983625
+ "step_time": 8.379146702999606
},
{
"clip_ratio/high_max": 0.0,
@@ -9620,26 +9620,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 51.25,
- "completions/mean_terminated_length": 51.25,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.1820043921470642,
- "epoch": 2.796747967479675,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1726708859205246,
- "kl": 2.1745819140051026e-05,
- "learning_rate": 1.2045047355789107e-08,
- "loss": 0.048721786588430405,
- "num_tokens": 1415472.0,
- "reward": 0.6344382762908936,
- "reward_std": 0.07378704100847244,
- "rewards/true_env_reward_fn/mean": 0.6344382762908936,
- "rewards/true_env_reward_fn/std": 0.07378704845905304,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 72.0625,
+ "completions/mean_terminated_length": 72.0625,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.365002065896988,
+ "epoch": 8.390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07387524098157883,
+ "kl": 7.906068458396476e-05,
+ "learning_rate": 7.91641364814628e-08,
+ "loss": -0.019427858293056488,
+ "num_tokens": 8502659.0,
+ "reward": 0.49987101554870605,
+ "reward_std": 0.22534418106079102,
+ "rewards/true_env_reward_fn/mean": 0.49987101554870605,
+ "rewards/true_env_reward_fn/std": 0.22534416615962982,
"step": 344,
- "step_time": 3.2691068980002456
+ "step_time": 16.447989433999282
},
{
"clip_ratio/high_max": 0.0,
@@ -9648,26 +9648,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 90.0,
- "completions/max_terminated_length": 90.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.0933732390403748,
- "epoch": 2.8048780487804876,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21739810705184937,
- "kl": 2.0197898265905678e-05,
- "learning_rate": 1.1140566232228677e-08,
- "loss": 0.0031427331268787384,
- "num_tokens": 1417385.0,
- "reward": 0.5309104323387146,
- "reward_std": 0.18906596302986145,
- "rewards/true_env_reward_fn/mean": 0.5309104323387146,
- "rewards/true_env_reward_fn/std": 0.18906594812870026,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 63.833335876464844,
+ "completions/mean_terminated_length": 63.833335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3033632636070251,
+ "epoch": 8.414634146341463,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07896492630243301,
+ "kl": 0.00015640226229152177,
+ "learning_rate": 7.688073445617798e-08,
+ "loss": 0.032075606286525726,
+ "num_tokens": 8527771.0,
+ "reward": 0.4142627418041229,
+ "reward_std": 0.39965084195137024,
+ "rewards/true_env_reward_fn/mean": 0.4142627418041229,
+ "rewards/true_env_reward_fn/std": 0.3996508717536926,
"step": 345,
- "step_time": 3.9515345389991126
+ "step_time": 10.625701701998878
},
{
"clip_ratio/high_max": 0.0,
@@ -9676,26 +9676,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 58.875,
- "completions/mean_terminated_length": 58.875,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "entropy": 1.2968629598617554,
- "epoch": 2.813008130081301,
+ "completions/max_length": 222.0,
+ "completions/max_terminated_length": 222.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2410016655921936,
+ "epoch": 8.439024390243903,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.20110434293746948,
- "kl": 4.813342457055114e-05,
- "learning_rate": 1.0270903842714673e-08,
- "loss": 0.062390223145484924,
- "num_tokens": 1423496.0,
- "reward": 0.14946666359901428,
- "reward_std": 0.24863645434379578,
- "rewards/true_env_reward_fn/mean": 0.14946666359901428,
- "rewards/true_env_reward_fn/std": 0.24863646924495697,
+ "grad_norm": 0.10202007740736008,
+ "kl": 0.00028172876773169264,
+ "learning_rate": 7.462800196578661e-08,
+ "loss": 0.030375666916370392,
+ "num_tokens": 8557085.0,
+ "reward": 0.3401162028312683,
+ "reward_std": 0.2810371518135071,
+ "rewards/true_env_reward_fn/mean": 0.3401161730289459,
+ "rewards/true_env_reward_fn/std": 0.2810371518135071,
"step": 346,
- "step_time": 4.331638548001138
+ "step_time": 15.773468264999792
},
{
"clip_ratio/high_max": 0.0,
@@ -9704,26 +9704,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.75,
- "completions/mean_terminated_length": 63.75,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "entropy": 1.153487741947174,
- "epoch": 2.821138211382114,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.00011695526336552575,
- "kl": 1.7154279703390785e-05,
- "learning_rate": 9.436138057399601e-09,
- "loss": 8.606326673543663e-07,
- "num_tokens": 1427182.0,
- "reward": 0.5776733160018921,
- "reward_std": 0.11168669909238815,
- "rewards/true_env_reward_fn/mean": 0.5776733160018921,
- "rewards/true_env_reward_fn/std": 0.11168669164180756,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 62.97916793823242,
+ "completions/mean_terminated_length": 62.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4290962517261505,
+ "epoch": 8.463414634146341,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06221809610724449,
+ "kl": 9.756034887686837e-05,
+ "learning_rate": 7.240610229819195e-08,
+ "loss": -0.009350121021270752,
+ "num_tokens": 8583644.0,
+ "reward": 0.42270374298095703,
+ "reward_std": 0.19876612722873688,
+ "rewards/true_env_reward_fn/mean": 0.42270374298095703,
+ "rewards/true_env_reward_fn/std": 0.19876612722873688,
"step": 347,
- "step_time": 3.416529175998221
+ "step_time": 10.18261290999908
},
{
"clip_ratio/high_max": 0.0,
@@ -9732,26 +9732,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.3210760354995728,
- "epoch": 2.8292682926829267,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13752248883247375,
- "kl": 3.656550507002976e-05,
- "learning_rate": 8.636343621770948e-09,
- "loss": -0.029253121465444565,
- "num_tokens": 1431839.0,
- "reward": 0.47378063201904297,
- "reward_std": 0.32937321066856384,
- "rewards/true_env_reward_fn/mean": 0.47378063201904297,
- "rewards/true_env_reward_fn/std": 0.32937324047088623,
+ "completions/max_length": 361.0,
+ "completions/max_terminated_length": 361.0,
+ "completions/mean_length": 68.89583587646484,
+ "completions/mean_terminated_length": 68.89583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2232879400253296,
+ "epoch": 8.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07711587101221085,
+ "kl": 7.56127965360065e-05,
+ "learning_rate": 7.021519650639951e-08,
+ "loss": -0.06937319040298462,
+ "num_tokens": 8604055.0,
+ "reward": 0.5483574867248535,
+ "reward_std": 0.2410479635000229,
+ "rewards/true_env_reward_fn/mean": 0.5483574867248535,
+ "rewards/true_env_reward_fn/std": 0.2410479635000229,
"step": 348,
- "step_time": 3.957806388998506
+ "step_time": 20.953229555999314
},
{
"clip_ratio/high_max": 0.0,
@@ -9760,26 +9760,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 129.0,
- "completions/max_terminated_length": 129.0,
- "completions/mean_length": 83.75,
- "completions/mean_terminated_length": 83.75,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "entropy": 1.3464251160621643,
- "epoch": 2.83739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11163587868213654,
- "kl": 1.8057865418086294e-05,
- "learning_rate": 7.87159214995774e-09,
- "loss": 0.01491672545671463,
- "num_tokens": 1434785.0,
- "reward": 0.6471105217933655,
- "reward_std": 0.1651550978422165,
- "rewards/true_env_reward_fn/mean": 0.6471105217933655,
- "rewards/true_env_reward_fn/std": 0.1651550978422165,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.2409018576145172,
+ "epoch": 8.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08634211868047714,
+ "kl": 0.000148440574776032,
+ "learning_rate": 6.805544339684293e-08,
+ "loss": 0.05703897029161453,
+ "num_tokens": 8636849.0,
+ "reward": 0.2619777023792267,
+ "reward_std": 0.3443836271762848,
+ "rewards/true_env_reward_fn/mean": 0.2619777023792267,
+ "rewards/true_env_reward_fn/std": 0.3443836271762848,
"step": 349,
- "step_time": 5.513293031999638
+ "step_time": 13.73512468699937
},
{
"clip_ratio/high_max": 0.0,
@@ -9788,32 +9788,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.0,
- "completions/mean_terminated_length": 57.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.1192307472229004,
- "epoch": 2.845528455284553,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20383992791175842,
- "kl": 1.563574414831237e-05,
- "learning_rate": 7.141952118318961e-09,
- "loss": 0.113973468542099,
- "num_tokens": 1437333.0,
- "reward": 0.5760529637336731,
- "reward_std": 0.11078595370054245,
- "rewards/true_env_reward_fn/mean": 0.5760529637336731,
- "rewards/true_env_reward_fn/std": 0.11078596115112305,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 64.85417175292969,
+ "completions/mean_terminated_length": 64.85417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2963348627090454,
+ "epoch": 8.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09569360315799713,
+ "kl": 0.0001818603413994424,
+ "learning_rate": 6.592699951787362e-08,
+ "loss": 0.0750311091542244,
+ "num_tokens": 8668938.0,
+ "reward": 0.2865946292877197,
+ "reward_std": 0.21787859499454498,
+ "rewards/true_env_reward_fn/mean": 0.2865946292877197,
+ "rewards/true_env_reward_fn/std": 0.21787859499454498,
"step": 350,
- "step_time": 3.624226386000373
+ "step_time": 16.08931997000036
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 1437333,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 8668938,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -9828,7 +9828,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-350/training_args.bin b/checkpoint-350/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-350/training_args.bin
+++ b/checkpoint-350/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/checkpoint-400/README.md b/checkpoint-400/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-400/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-400/adapter_config.json b/checkpoint-400/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-400/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-400/adapter_model.safetensors b/checkpoint-400/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3f5b76a9b7bf1eefd34a61ed093e9ff8ca9b3df5
--- /dev/null
+++ b/checkpoint-400/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99cf1bd568562586b3103f6771c463e2ce41da9fa48e2842bafabe58603069a8
+size 8731128
diff --git a/checkpoint-400/chat_template.jinja b/checkpoint-400/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-400/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-400/optimizer.pt b/checkpoint-400/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc77df6c0e3c15f88955db8fdb230900f81521d6
--- /dev/null
+++ b/checkpoint-400/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b530df27dedb971c46aa262acd525d1ffbb3f7cb5410e745d90c698778edd46
+size 17526842
diff --git a/checkpoint-400/ref/adapter_config.json b/checkpoint-400/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-400/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-400/ref/adapter_model.safetensors b/checkpoint-400/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4b516b95e2bde01b4a51b7977bce639f00946144
--- /dev/null
+++ b/checkpoint-400/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
+size 8731128
diff --git a/checkpoint-400/rng_state.pth b/checkpoint-400/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4dee724664fc6727f4e832a2f17014b66f6d8fdd
--- /dev/null
+++ b/checkpoint-400/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aaaf98b53baee8d092262bd3cc662d4f79d16730605a0cc9b41a5aa00eda454
+size 14244
diff --git a/checkpoint-400/scaler.pt b/checkpoint-400/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..834fccb9e03a3266c967e2aa2ac0269bf2253fcc
--- /dev/null
+++ b/checkpoint-400/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c801982aae9be06d302403c1fff693e53dedf89c1d3b689ee29fedad84a96d23
+size 988
diff --git a/checkpoint-400/scheduler.pt b/checkpoint-400/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0d96232c10a33e5c54aa176a3b9d660575569ed
--- /dev/null
+++ b/checkpoint-400/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33455f235d7ec3294419a8b99f18cb36a722087dfde6eba363e622865c3dcf0c
+size 1064
diff --git a/checkpoint-400/tokenizer.json b/checkpoint-400/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-400/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-400/tokenizer_config.json b/checkpoint-400/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-400/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-400/trainer_state.json b/checkpoint-400/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..81ee694902890e687dc77bbd64de79ecae7db8dc
--- /dev/null
+++ b/checkpoint-400/trainer_state.json
@@ -0,0 +1,11234 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 9.75609756097561,
+ "eval_steps": 500,
+ "global_step": 400,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
+ "learning_rate": 0.0,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
+ "step": 1,
+ "step_time": 11.815711200999885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
+ "step": 2,
+ "step_time": 13.493524850000085
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
+ "step": 3,
+ "step_time": 11.449303891999875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
+ "step": 4,
+ "step_time": 18.835909622000145
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
+ "step": 5,
+ "step_time": 14.435845696000001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
+ "step": 6,
+ "step_time": 13.232063896999989
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
+ "step": 7,
+ "step_time": 9.810652986000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
+ "step": 8,
+ "step_time": 19.874756868999953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
+ "step": 9,
+ "step_time": 9.458149736999985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
+ "step": 10,
+ "step_time": 14.219840567000006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
+ "step": 11,
+ "step_time": 9.271131832999913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
+ "step": 12,
+ "step_time": 12.184364300000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
+ "step": 13,
+ "step_time": 10.720424850000086
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
+ "step": 14,
+ "step_time": 14.965493325000011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
+ "step": 15,
+ "step_time": 13.628413805999912
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
+ "step": 16,
+ "step_time": 14.229579036999894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
+ "step": 17,
+ "step_time": 17.023353198999985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
+ "step": 18,
+ "step_time": 11.965533113999868
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
+ "step": 19,
+ "step_time": 10.33812468799988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
+ "step": 20,
+ "step_time": 17.317542748000164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
+ "step": 21,
+ "step_time": 12.20283881399996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
+ "step": 22,
+ "step_time": 11.211206898000114
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
+ "step": 23,
+ "step_time": 10.428452587999914
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
+ "step": 24,
+ "step_time": 11.223491792000118
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
+ "step": 25,
+ "step_time": 10.92509102200006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
+ "step": 26,
+ "step_time": 10.458724108999945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
+ "step": 27,
+ "step_time": 14.364785926000081
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
+ "step": 28,
+ "step_time": 10.888908384999922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
+ "step": 29,
+ "step_time": 15.510035302999995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
+ "step": 30,
+ "step_time": 14.50245602599989
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
+ "step": 31,
+ "step_time": 11.06186091799998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
+ "step": 32,
+ "step_time": 15.808748693000211
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
+ "step": 33,
+ "step_time": 24.20358999299981
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
+ "step": 34,
+ "step_time": 12.218407348999904
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
+ "step": 35,
+ "step_time": 9.66908789599995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
+ "step": 36,
+ "step_time": 8.370980583999994
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
+ "step": 37,
+ "step_time": 10.285125336999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
+ "step": 38,
+ "step_time": 12.449738128000035
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
+ "step": 39,
+ "step_time": 10.279209028999958
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
+ "step": 40,
+ "step_time": 13.58010066599968
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
+ "step": 41,
+ "step_time": 10.35077203700007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
+ "step": 42,
+ "step_time": 10.548370664999993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
+ "step": 43,
+ "step_time": 24.089396637999698
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
+ "step": 44,
+ "step_time": 19.39071501599983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
+ "step": 45,
+ "step_time": 13.893569495999827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
+ "step": 46,
+ "step_time": 10.50698806499986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
+ "step": 47,
+ "step_time": 12.964419044000124
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
+ "step": 48,
+ "step_time": 17.23041258299986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
+ "step": 49,
+ "step_time": 8.691208415999881
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
+ "step": 50,
+ "step_time": 10.162109979000206
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
+ "step": 51,
+ "step_time": 10.88732858200001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
+ "step": 52,
+ "step_time": 9.252596976000177
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
+ "step": 53,
+ "step_time": 24.22518693999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
+ "step": 54,
+ "step_time": 10.14821418500037
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
+ "step": 55,
+ "step_time": 11.13082981000025
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
+ "step": 56,
+ "step_time": 14.49393488900023
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
+ "step": 57,
+ "step_time": 14.065935286000013
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
+ "step": 58,
+ "step_time": 12.76291504100027
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
+ "step": 59,
+ "step_time": 11.089177285999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
+ "step": 60,
+ "step_time": 12.328215124000053
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
+ "step": 61,
+ "step_time": 13.294195681999781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
+ "step": 62,
+ "step_time": 18.972790307999958
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
+ "step": 63,
+ "step_time": 11.292631820999986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
+ "step": 64,
+ "step_time": 9.600786530999585
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
+ "step": 65,
+ "step_time": 10.169144185999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
+ "step": 66,
+ "step_time": 10.343706631000032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
+ "step": 67,
+ "step_time": 12.16719347000003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
+ "step": 68,
+ "step_time": 15.523659553000016
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
+ "step": 69,
+ "step_time": 9.677535032999685
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
+ "step": 70,
+ "step_time": 15.20990351499995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
+ "step": 71,
+ "step_time": 14.703903473999844
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
+ "step": 72,
+ "step_time": 9.422353613000269
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
+ "step": 73,
+ "step_time": 16.65922043799992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
+ "step": 74,
+ "step_time": 13.422026366999944
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
+ "step": 75,
+ "step_time": 9.51117546200021
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
+ "step": 76,
+ "step_time": 9.295928349999713
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
+ "step": 77,
+ "step_time": 15.693113021000045
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
+ "step": 78,
+ "step_time": 16.146651725000083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
+ "step": 79,
+ "step_time": 9.82867347299998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
+ "step": 80,
+ "step_time": 11.267316974000096
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
+ "step": 81,
+ "step_time": 10.035370067999793
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
+ "step": 82,
+ "step_time": 9.56242024800008
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
+ "step": 83,
+ "step_time": 10.563381390999666
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
+ "step": 84,
+ "step_time": 17.21621736799966
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
+ "step": 85,
+ "step_time": 25.519813745999954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
+ "step": 86,
+ "step_time": 10.78625990699993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
+ "step": 87,
+ "step_time": 13.604215705999877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
+ "step": 88,
+ "step_time": 32.408574500999975
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
+ "step": 89,
+ "step_time": 9.901715897000031
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
+ "step": 90,
+ "step_time": 11.152492722000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
+ "step": 91,
+ "step_time": 10.656350811000038
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
+ "step": 92,
+ "step_time": 10.304143018000104
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
+ "step": 93,
+ "step_time": 13.589426085000014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
+ "step": 94,
+ "step_time": 14.587356482000132
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
+ "step": 95,
+ "step_time": 14.689755582000089
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
+ "step": 96,
+ "step_time": 10.379233056999965
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
+ "step": 97,
+ "step_time": 10.037491584999998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
+ "step": 98,
+ "step_time": 13.073343929999965
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
+ "step": 99,
+ "step_time": 9.465850557000067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
+ "step": 100,
+ "step_time": 11.33050741000011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
+ "step": 101,
+ "step_time": 16.478299477000064
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
+ "step": 102,
+ "step_time": 11.344593008000174
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
+ "step": 103,
+ "step_time": 8.135681302999728
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
+ "step": 104,
+ "step_time": 15.185034105999875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
+ "step": 105,
+ "step_time": 9.588520330999927
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
+ "step": 106,
+ "step_time": 14.60399662399982
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
+ "step": 107,
+ "step_time": 20.100954443999854
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
+ "step": 108,
+ "step_time": 8.304149297999857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
+ "step": 109,
+ "step_time": 11.260021517000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
+ "step": 110,
+ "step_time": 22.80178854000019
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
+ "step": 111,
+ "step_time": 9.663163859000178
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
+ "step": 112,
+ "step_time": 9.903081222999617
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
+ "step": 113,
+ "step_time": 10.101770388999284
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
+ "step": 114,
+ "step_time": 17.633509853000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
+ "step": 115,
+ "step_time": 12.307247350000125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
+ "step": 116,
+ "step_time": 14.468690254000194
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
+ "step": 117,
+ "step_time": 11.849063975999798
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
+ "step": 118,
+ "step_time": 16.597334930000216
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
+ "step": 119,
+ "step_time": 11.729475523999554
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
+ "step": 120,
+ "step_time": 11.503627788000358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
+ "step": 121,
+ "step_time": 8.728293746000418
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
+ "step": 122,
+ "step_time": 15.51957702100026
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
+ "step": 123,
+ "step_time": 8.471463828000196
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
+ "step": 124,
+ "step_time": 13.438758649999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
+ "step": 125,
+ "step_time": 8.654177170999901
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
+ "step": 126,
+ "step_time": 10.513378469000145
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
+ "step": 127,
+ "step_time": 23.20779430600078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
+ "step": 128,
+ "step_time": 11.551069149999876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
+ "step": 129,
+ "step_time": 10.723005456000465
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
+ "step": 130,
+ "step_time": 15.15810051499966
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
+ "step": 131,
+ "step_time": 20.109428818001106
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
+ "step": 132,
+ "step_time": 10.12403799699996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
+ "step": 133,
+ "step_time": 10.396350653999434
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
+ "step": 134,
+ "step_time": 10.961974539000039
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
+ "step": 135,
+ "step_time": 11.779171687000144
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
+ "step": 136,
+ "step_time": 8.779588141000204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
+ "step": 137,
+ "step_time": 11.74765996799988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
+ "step": 138,
+ "step_time": 8.8135579650002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
+ "step": 139,
+ "step_time": 9.997661417000472
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
+ "step": 140,
+ "step_time": 14.679971276999822
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
+ "step": 141,
+ "step_time": 11.64747691499997
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
+ "step": 142,
+ "step_time": 8.380270293999729
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
+ "step": 143,
+ "step_time": 11.716556537000088
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
+ "step": 144,
+ "step_time": 10.67718802499985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
+ "step": 145,
+ "step_time": 8.610043666999445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
+ "step": 146,
+ "step_time": 13.649344002000362
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
+ "step": 147,
+ "step_time": 11.702765863000877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
+ "step": 148,
+ "step_time": 13.306644664000487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
+ "step": 149,
+ "step_time": 9.446422488999815
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
+ "step": 150,
+ "step_time": 10.538116119999813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
+ "step": 151,
+ "step_time": 15.755764130999978
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
+ "step": 152,
+ "step_time": 9.670861957999932
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
+ "step": 153,
+ "step_time": 22.82054339000024
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
+ "step": 154,
+ "step_time": 9.973958625999785
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
+ "step": 155,
+ "step_time": 9.758407419000378
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
+ "step": 156,
+ "step_time": 11.009583763999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
+ "step": 157,
+ "step_time": 10.12789283499933
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
+ "step": 158,
+ "step_time": 7.800485663000472
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
+ "step": 159,
+ "step_time": 13.77649076500029
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
+ "step": 160,
+ "step_time": 13.25029361300085
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
+ "step": 161,
+ "step_time": 17.837881629000094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
+ "step": 162,
+ "step_time": 9.286757633000889
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
+ "step": 163,
+ "step_time": 10.785562561999996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
+ "step": 164,
+ "step_time": 12.722446307000155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
+ "step": 165,
+ "step_time": 14.89655208000022
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
+ "step": 166,
+ "step_time": 8.962532588000613
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
+ "step": 167,
+ "step_time": 11.708963983000103
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
+ "step": 168,
+ "step_time": 16.905241970000134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
+ "step": 169,
+ "step_time": 12.27045077799994
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
+ "step": 170,
+ "step_time": 31.721865252000498
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
+ "step": 171,
+ "step_time": 8.973740739000277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
+ "step": 172,
+ "step_time": 10.39117795600032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
+ "step": 173,
+ "step_time": 11.8111169230001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
+ "step": 174,
+ "step_time": 17.211099596999702
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
+ "step": 175,
+ "step_time": 12.29185969800028
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
+ "step": 176,
+ "step_time": 10.987576109999736
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
+ "step": 177,
+ "step_time": 10.792315139000493
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
+ "step": 178,
+ "step_time": 12.399353334999887
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
+ "step": 179,
+ "step_time": 12.45691162600042
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
+ "step": 180,
+ "step_time": 12.108760526000424
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
+ "step": 181,
+ "step_time": 14.16400909000049
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
+ "step": 182,
+ "step_time": 19.767916835999586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
+ "step": 183,
+ "step_time": 10.428195530999346
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
+ "step": 184,
+ "step_time": 16.47171987999991
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
+ "step": 185,
+ "step_time": 13.684267182999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
+ "step": 186,
+ "step_time": 10.281018189999031
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
+ "step": 187,
+ "step_time": 24.351223329000277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
+ "step": 188,
+ "step_time": 10.644911742000204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
+ "step": 189,
+ "step_time": 9.579594637999435
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
+ "step": 190,
+ "step_time": 11.495368679999501
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
+ "step": 191,
+ "step_time": 7.909104242000012
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
+ "step": 192,
+ "step_time": 10.043768619000275
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
+ "step": 193,
+ "step_time": 10.19648474899941
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
+ "step": 194,
+ "step_time": 14.372816425000565
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
+ "step": 195,
+ "step_time": 11.282298853000157
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
+ "step": 196,
+ "step_time": 8.65745804199969
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
+ "step": 197,
+ "step_time": 9.615110594999805
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
+ "step": 198,
+ "step_time": 10.832162847999825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
+ "step": 199,
+ "step_time": 10.94759418800004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
+ "step": 200,
+ "step_time": 10.322844021000492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.083335876464844,
+ "completions/mean_terminated_length": 62.083335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.319727510213852,
+ "epoch": 4.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08774503320455551,
+ "kl": 6.86226876496221e-05,
+ "learning_rate": 6.077002463994907e-07,
+ "loss": 0.058894164860248566,
+ "num_tokens": 4956466.0,
+ "reward": 0.41384777426719666,
+ "reward_std": 0.17193447053432465,
+ "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+ "rewards/true_env_reward_fn/std": 0.17193445563316345,
+ "step": 201,
+ "step_time": 10.146928047999609
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 59.395835876464844,
+ "completions/mean_terminated_length": 59.395835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1935442388057709,
+ "epoch": 4.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08328510820865631,
+ "kl": 7.948942857183283e-05,
+ "learning_rate": 6.035394200708103e-07,
+ "loss": 0.045964501798152924,
+ "num_tokens": 4978733.0,
+ "reward": 0.4896667003631592,
+ "reward_std": 0.3830767571926117,
+ "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+ "rewards/true_env_reward_fn/std": 0.3830767571926117,
+ "step": 202,
+ "step_time": 10.025533761000133
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 62.22916793823242,
+ "completions/mean_terminated_length": 62.22916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2698631286621094,
+ "epoch": 4.951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07158155739307404,
+ "kl": 8.286665615742095e-05,
+ "learning_rate": 5.993710887521302e-07,
+ "loss": -0.03551984950900078,
+ "num_tokens": 5009440.0,
+ "reward": 0.45639634132385254,
+ "reward_std": 0.1837630569934845,
+ "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+ "rewards/true_env_reward_fn/std": 0.1837630420923233,
+ "step": 203,
+ "step_time": 9.823523802999716
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2825455367565155,
+ "epoch": 4.975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07431793212890625,
+ "kl": 7.247529447340639e-05,
+ "learning_rate": 5.951955545823342e-07,
+ "loss": -0.02418600022792816,
+ "num_tokens": 5036036.0,
+ "reward": 0.4476773142814636,
+ "reward_std": 0.20447416603565216,
+ "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+ "rewards/true_env_reward_fn/std": 0.20447418093681335,
+ "step": 204,
+ "step_time": 11.960790695000014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2438389956951141,
+ "epoch": 5.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06833480298519135,
+ "kl": 5.646793260893901e-05,
+ "learning_rate": 5.91013120222401e-07,
+ "loss": 0.0021926667541265488,
+ "num_tokens": 5063228.0,
+ "reward": 0.4300723075866699,
+ "reward_std": 0.12116922438144684,
+ "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+ "rewards/true_env_reward_fn/std": 0.12116922438144684,
+ "step": 205,
+ "step_time": 9.874485716000436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 205.0,
+ "completions/max_terminated_length": 205.0,
+ "completions/mean_length": 66.1875,
+ "completions/mean_terminated_length": 66.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2645181119441986,
+ "epoch": 5.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08248262107372284,
+ "kl": 9.333990783488844e-05,
+ "learning_rate": 5.868240888334652e-07,
+ "loss": 0.12253005057573318,
+ "num_tokens": 5093373.0,
+ "reward": 0.28542038798332214,
+ "reward_std": 0.326623797416687,
+ "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+ "rewards/true_env_reward_fn/std": 0.326623797416687,
+ "step": 206,
+ "step_time": 15.706792760000553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.16667175292969,
+ "completions/mean_terminated_length": 67.16667175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.34083291888237,
+ "epoch": 5.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07931552827358246,
+ "kl": 0.00010715152166085318,
+ "learning_rate": 5.826287640548424e-07,
+ "loss": 0.0005592256784439087,
+ "num_tokens": 5118933.0,
+ "reward": 0.5035215616226196,
+ "reward_std": 0.304157018661499,
+ "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+ "rewards/true_env_reward_fn/std": 0.30415698885917664,
+ "step": 207,
+ "step_time": 10.04168460900064
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 73.4375,
+ "completions/mean_terminated_length": 73.4375,
+ "completions/min_length": 11.0,
+ "completions/min_terminated_length": 11.0,
+ "entropy": 1.1821868121623993,
+ "epoch": 5.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07619761675596237,
+ "kl": 0.00013189401397539768,
+ "learning_rate": 5.784274499820213e-07,
+ "loss": -0.011967688798904419,
+ "num_tokens": 5148682.0,
+ "reward": 0.36097532510757446,
+ "reward_std": 0.29208436608314514,
+ "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+ "rewards/true_env_reward_fn/std": 0.29208436608314514,
+ "step": 208,
+ "step_time": 15.226898961000188
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 75.39583587646484,
+ "completions/mean_terminated_length": 75.39583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2636725008487701,
+ "epoch": 5.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06586338579654694,
+ "kl": 1.9426477138040354e-05,
+ "learning_rate": 5.742204511446203e-07,
+ "loss": -0.047095417976379395,
+ "num_tokens": 5173205.0,
+ "reward": 0.5261925458908081,
+ "reward_std": 0.2689943313598633,
+ "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+ "rewards/true_env_reward_fn/std": 0.2689943313598633,
+ "step": 209,
+ "step_time": 23.690397457000017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 54.3125,
+ "completions/mean_terminated_length": 54.3125,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2415392696857452,
+ "epoch": 5.121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0990133062005043,
+ "kl": 7.119746442185715e-05,
+ "learning_rate": 5.700080724843146e-07,
+ "loss": -0.030588299036026,
+ "num_tokens": 5199892.0,
+ "reward": 0.3418487012386322,
+ "reward_std": 0.36353805661201477,
+ "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+ "rewards/true_env_reward_fn/std": 0.3635380268096924,
+ "step": 210,
+ "step_time": 9.864614251999683
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2508135735988617,
+ "epoch": 5.146341463414634,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10835102945566177,
+ "kl": 0.00012395972225931473,
+ "learning_rate": 5.657906193327324e-07,
+ "loss": 0.08621911704540253,
+ "num_tokens": 5220141.0,
+ "reward": 0.4987506866455078,
+ "reward_std": 0.27843451499938965,
+ "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+ "rewards/true_env_reward_fn/std": 0.27843451499938965,
+ "step": 211,
+ "step_time": 9.983622502000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1611086130142212,
+ "epoch": 5.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08423946797847748,
+ "kl": 4.4347343191475375e-05,
+ "learning_rate": 5.615683973893234e-07,
+ "loss": 0.08670675754547119,
+ "num_tokens": 5244470.0,
+ "reward": 0.4252437949180603,
+ "reward_std": 0.2996494472026825,
+ "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+ "rewards/true_env_reward_fn/std": 0.2996494472026825,
+ "step": 212,
+ "step_time": 18.68646409699977
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3110275566577911,
+ "epoch": 5.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06817487627267838,
+ "kl": 0.00010393545744591393,
+ "learning_rate": 5.573417126992002e-07,
+ "loss": 0.04062439873814583,
+ "num_tokens": 5271369.0,
+ "reward": 0.4340522885322571,
+ "reward_std": 0.26940545439720154,
+ "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+ "rewards/true_env_reward_fn/std": 0.26940542459487915,
+ "step": 213,
+ "step_time": 11.3590317649996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 54.4375,
+ "completions/mean_terminated_length": 54.4375,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2018881738185883,
+ "epoch": 5.219512195121951,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10059589147567749,
+ "kl": 0.00011373830420779996,
+ "learning_rate": 5.531108716309547e-07,
+ "loss": -0.09816272556781769,
+ "num_tokens": 5288638.0,
+ "reward": 0.5691710710525513,
+ "reward_std": 0.25253745913505554,
+ "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+ "rewards/true_env_reward_fn/std": 0.25253745913505554,
+ "step": 214,
+ "step_time": 9.633293675000004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1958912312984467,
+ "epoch": 5.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0779944583773613,
+ "kl": 7.025236300250981e-05,
+ "learning_rate": 5.488761808544509e-07,
+ "loss": 0.03371567651629448,
+ "num_tokens": 5312081.0,
+ "reward": 0.44519662857055664,
+ "reward_std": 0.22201679646968842,
+ "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+ "rewards/true_env_reward_fn/std": 0.22201678156852722,
+ "step": 215,
+ "step_time": 12.463993090999338
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.2146256864070892,
+ "epoch": 5.2682926829268295,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06834157556295395,
+ "kl": 0.00010125362859980669,
+ "learning_rate": 5.446379473185971e-07,
+ "loss": -0.02198379673063755,
+ "num_tokens": 5334238.0,
+ "reward": 0.5273472666740417,
+ "reward_std": 0.21954773366451263,
+ "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+ "rewards/true_env_reward_fn/std": 0.21954771876335144,
+ "step": 216,
+ "step_time": 15.126136884000061
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 61.333335876464844,
+ "completions/mean_terminated_length": 61.333335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2872387170791626,
+ "epoch": 5.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09578828513622284,
+ "kl": 7.404103780572768e-05,
+ "learning_rate": 5.403964782290961e-07,
+ "loss": -0.041091397404670715,
+ "num_tokens": 5351798.0,
+ "reward": 0.5464547872543335,
+ "reward_std": 0.16224616765975952,
+ "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+ "rewards/true_env_reward_fn/std": 0.16224615275859833,
+ "step": 217,
+ "step_time": 7.2901647220001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 84.89583587646484,
+ "completions/mean_terminated_length": 84.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.4322342276573181,
+ "epoch": 5.317073170731708,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0667513981461525,
+ "kl": 2.497344303264981e-05,
+ "learning_rate": 5.361520810261778e-07,
+ "loss": -0.06392769515514374,
+ "num_tokens": 5375369.0,
+ "reward": 0.5213420391082764,
+ "reward_std": 0.1436246931552887,
+ "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+ "rewards/true_env_reward_fn/std": 0.1436246782541275,
+ "step": 218,
+ "step_time": 12.217936152999755
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2382279634475708,
+ "epoch": 5.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07372384518384933,
+ "kl": 9.740726909512887e-05,
+ "learning_rate": 5.319050633623141e-07,
+ "loss": 0.010912742465734482,
+ "num_tokens": 5397956.0,
+ "reward": 0.5623860359191895,
+ "reward_std": 0.285375714302063,
+ "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+ "rewards/true_env_reward_fn/std": 0.285375714302063,
+ "step": 219,
+ "step_time": 13.449634822999997
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 65.39583587646484,
+ "completions/mean_terminated_length": 65.39583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2669419348239899,
+ "epoch": 5.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07886125147342682,
+ "kl": 0.00011165817522851285,
+ "learning_rate": 5.276557330799203e-07,
+ "loss": -0.02433733269572258,
+ "num_tokens": 5418511.0,
+ "reward": 0.6075118780136108,
+ "reward_std": 0.30748677253723145,
+ "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+ "rewards/true_env_reward_fn/std": 0.30748677253723145,
+ "step": 220,
+ "step_time": 12.051496982000117
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2560602128505707,
+ "epoch": 5.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08607357740402222,
+ "kl": 8.424731549894204e-05,
+ "learning_rate": 5.234043981890393e-07,
+ "loss": 0.013634156435728073,
+ "num_tokens": 5439690.0,
+ "reward": 0.5080039501190186,
+ "reward_std": 0.21975299715995789,
+ "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+ "rewards/true_env_reward_fn/std": 0.21975299715995789,
+ "step": 221,
+ "step_time": 10.435893627000496
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3081265985965729,
+ "epoch": 5.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0876006931066513,
+ "kl": 9.127605972025776e-05,
+ "learning_rate": 5.191513668450177e-07,
+ "loss": -0.01968565583229065,
+ "num_tokens": 5468216.0,
+ "reward": 0.3679848611354828,
+ "reward_std": 0.237859845161438,
+ "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+ "rewards/true_env_reward_fn/std": 0.237859845161438,
+ "step": 222,
+ "step_time": 11.524256381999294
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 67.66667175292969,
+ "completions/mean_terminated_length": 67.66667175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2990808486938477,
+ "epoch": 5.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07685395330190659,
+ "kl": 0.00014209141045284923,
+ "learning_rate": 5.148969473261679e-07,
+ "loss": -0.01107364147901535,
+ "num_tokens": 5488040.0,
+ "reward": 0.5435788035392761,
+ "reward_std": 0.35419100522994995,
+ "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+ "rewards/true_env_reward_fn/std": 0.35419100522994995,
+ "step": 223,
+ "step_time": 11.945272217000365
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.284348964691162,
+ "epoch": 5.463414634146342,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.058678388595581055,
+ "kl": 3.3754420201148605e-05,
+ "learning_rate": 5.106414480114238e-07,
+ "loss": -0.01714605651795864,
+ "num_tokens": 5513005.0,
+ "reward": 0.43836766481399536,
+ "reward_std": 0.26744046807289124,
+ "rewards/true_env_reward_fn/mean": 0.438367635011673,
+ "rewards/true_env_reward_fn/std": 0.26744046807289124,
+ "step": 224,
+ "step_time": 10.563708176000091
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3357127904891968,
+ "epoch": 5.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07269197702407837,
+ "kl": 3.727909597728285e-05,
+ "learning_rate": 5.063851773579869e-07,
+ "loss": -0.01076439768075943,
+ "num_tokens": 5530655.0,
+ "reward": 0.6185358762741089,
+ "reward_std": 0.19721543788909912,
+ "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+ "rewards/true_env_reward_fn/std": 0.19721543788909912,
+ "step": 225,
+ "step_time": 10.372150705000422
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 64.35417175292969,
+ "completions/mean_terminated_length": 64.35417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2975924611091614,
+ "epoch": 5.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0875314474105835,
+ "kl": 9.186910938296933e-05,
+ "learning_rate": 5.021284438789693e-07,
+ "loss": 0.048179637640714645,
+ "num_tokens": 5554032.0,
+ "reward": 0.45552024245262146,
+ "reward_std": 0.31553781032562256,
+ "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+ "rewards/true_env_reward_fn/std": 0.31553778052330017,
+ "step": 226,
+ "step_time": 12.18860100899974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 63.208335876464844,
+ "completions/mean_terminated_length": 63.208335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2268341779708862,
+ "epoch": 5.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08318481594324112,
+ "kl": 8.052505017985823e-05,
+ "learning_rate": 4.978715561210307e-07,
+ "loss": 0.027963606640696526,
+ "num_tokens": 5582898.0,
+ "reward": 0.36310288310050964,
+ "reward_std": 0.3131144344806671,
+ "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+ "rewards/true_env_reward_fn/std": 0.3131144344806671,
+ "step": 227,
+ "step_time": 12.510411257000214
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 64.9375,
+ "completions/mean_terminated_length": 64.9375,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1668500006198883,
+ "epoch": 5.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0727877989411354,
+ "kl": 7.824771091691218e-05,
+ "learning_rate": 4.936148226420132e-07,
+ "loss": -0.01629078947007656,
+ "num_tokens": 5604791.0,
+ "reward": 0.5653349161148071,
+ "reward_std": 0.2849632799625397,
+ "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+ "rewards/true_env_reward_fn/std": 0.2849632799625397,
+ "step": 228,
+ "step_time": 10.083805716999905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 68.29167175292969,
+ "completions/mean_terminated_length": 68.29167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.307439923286438,
+ "epoch": 5.585365853658536,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06797961890697479,
+ "kl": 6.601300174224889e-05,
+ "learning_rate": 4.893585519885763e-07,
+ "loss": -0.036383360624313354,
+ "num_tokens": 5631629.0,
+ "reward": 0.49251794815063477,
+ "reward_std": 0.22127023339271545,
+ "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+ "rewards/true_env_reward_fn/std": 0.22127023339271545,
+ "step": 229,
+ "step_time": 13.650024606999523
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3524385392665863,
+ "epoch": 5.609756097560975,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.12484809756278992,
+ "kl": 0.0002558713749749586,
+ "learning_rate": 4.85103052673832e-07,
+ "loss": -0.0045075975358486176,
+ "num_tokens": 5666712.0,
+ "reward": 0.2605232000350952,
+ "reward_std": 0.22926004230976105,
+ "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+ "rewards/true_env_reward_fn/std": 0.22926005721092224,
+ "step": 230,
+ "step_time": 15.322059910999542
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2202666401863098,
+ "epoch": 5.634146341463414,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07976125925779343,
+ "kl": 8.684267686476232e-05,
+ "learning_rate": 4.808486331549823e-07,
+ "loss": 0.012383833527565002,
+ "num_tokens": 5702004.0,
+ "reward": 0.20571085810661316,
+ "reward_std": 0.2265808880329132,
+ "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+ "rewards/true_env_reward_fn/std": 0.2265808880329132,
+ "step": 231,
+ "step_time": 14.180213787999492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 312.0,
+ "completions/max_terminated_length": 312.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3096380531787872,
+ "epoch": 5.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08384906500577927,
+ "kl": 8.733692084206268e-05,
+ "learning_rate": 4.7659560181096067e-07,
+ "loss": 0.02098608762025833,
+ "num_tokens": 5729780.0,
+ "reward": 0.4599723219871521,
+ "reward_std": 0.21680118143558502,
+ "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+ "rewards/true_env_reward_fn/std": 0.21680118143558502,
+ "step": 232,
+ "step_time": 21.68401394800003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 53.520835876464844,
+ "completions/mean_terminated_length": 53.520835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2158988416194916,
+ "epoch": 5.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08225176483392715,
+ "kl": 9.274652438762132e-05,
+ "learning_rate": 4.7234426692007977e-07,
+ "loss": -0.034079790115356445,
+ "num_tokens": 5753397.0,
+ "reward": 0.5273070335388184,
+ "reward_std": 0.2194610834121704,
+ "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+ "rewards/true_env_reward_fn/std": 0.21946106851100922,
+ "step": 233,
+ "step_time": 7.915307780000603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 78.0,
+ "completions/mean_terminated_length": 78.0,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2362721860408783,
+ "epoch": 5.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06290840357542038,
+ "kl": 7.484563047910342e-05,
+ "learning_rate": 4.6809493663768575e-07,
+ "loss": -0.008873865008354187,
+ "num_tokens": 5778869.0,
+ "reward": 0.5283026695251465,
+ "reward_std": 0.195759579539299,
+ "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+ "rewards/true_env_reward_fn/std": 0.195759579539299,
+ "step": 234,
+ "step_time": 10.278297286999987
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 64.47917175292969,
+ "completions/mean_terminated_length": 64.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3364675641059875,
+ "epoch": 5.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09685558825731277,
+ "kl": 0.00013002969353692606,
+ "learning_rate": 4.638479189738224e-07,
+ "loss": 0.05070740357041359,
+ "num_tokens": 5804436.0,
+ "reward": 0.44921523332595825,
+ "reward_std": 0.22693434357643127,
+ "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+ "rewards/true_env_reward_fn/std": 0.22693434357643127,
+ "step": 235,
+ "step_time": 10.754199091999908
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2267529368400574,
+ "epoch": 5.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10451620817184448,
+ "kl": 6.754635069228243e-05,
+ "learning_rate": 4.596035217709039e-07,
+ "loss": -0.02925686165690422,
+ "num_tokens": 5826895.0,
+ "reward": 0.49523353576660156,
+ "reward_std": 0.15093794465065002,
+ "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+ "rewards/true_env_reward_fn/std": 0.15093792974948883,
+ "step": 236,
+ "step_time": 9.666070583999954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 67.79167175292969,
+ "completions/mean_terminated_length": 67.79167175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2110519707202911,
+ "epoch": 5.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07890205830335617,
+ "kl": 0.0001245876064785989,
+ "learning_rate": 4.5536205268140286e-07,
+ "loss": -0.03693925589323044,
+ "num_tokens": 5859973.0,
+ "reward": 0.2979053258895874,
+ "reward_std": 0.23015481233596802,
+ "rewards/true_env_reward_fn/mean": 0.297905296087265,
+ "rewards/true_env_reward_fn/std": 0.2301548272371292,
+ "step": 237,
+ "step_time": 13.971699990999241
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 65.52083587646484,
+ "completions/mean_terminated_length": 65.52083587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2291057407855988,
+ "epoch": 5.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09152546525001526,
+ "kl": 0.00013101351214572787,
+ "learning_rate": 4.511238191455491e-07,
+ "loss": 0.043641336262226105,
+ "num_tokens": 5886118.0,
+ "reward": 0.41334670782089233,
+ "reward_std": 0.19480590522289276,
+ "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+ "rewards/true_env_reward_fn/std": 0.19480590522289276,
+ "step": 238,
+ "step_time": 10.805698846999803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 58.833335876464844,
+ "completions/mean_terminated_length": 58.833335876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3134913444519043,
+ "epoch": 5.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07343924045562744,
+ "kl": 0.00019655993492051493,
+ "learning_rate": 4.4688912836904533e-07,
+ "loss": 0.015309082344174385,
+ "num_tokens": 5908334.0,
+ "reward": 0.4438478946685791,
+ "reward_std": 0.27188754081726074,
+ "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+ "rewards/true_env_reward_fn/std": 0.27188754081726074,
+ "step": 239,
+ "step_time": 9.688736522
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 464.0,
+ "completions/max_terminated_length": 464.0,
+ "completions/mean_length": 79.14583587646484,
+ "completions/mean_terminated_length": 79.14583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3148745000362396,
+ "epoch": 5.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06567981839179993,
+ "kl": 4.445325248525478e-05,
+ "learning_rate": 4.4265828730079977e-07,
+ "loss": 0.021218176931142807,
+ "num_tokens": 5929645.0,
+ "reward": 0.5866342186927795,
+ "reward_std": 0.13780196011066437,
+ "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+ "rewards/true_env_reward_fn/std": 0.13780196011066437,
+ "step": 240,
+ "step_time": 27.046819901000163
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 364.0,
+ "completions/max_terminated_length": 364.0,
+ "completions/mean_length": 70.33333587646484,
+ "completions/mean_terminated_length": 70.33333587646484,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2104995846748352,
+ "epoch": 5.878048780487805,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.042626503854990005,
+ "kl": 6.413207393052289e-05,
+ "learning_rate": 4.3843160261067653e-07,
+ "loss": -0.0065308245830237865,
+ "num_tokens": 5950005.0,
+ "reward": 0.5593677759170532,
+ "reward_std": 0.23993276059627533,
+ "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+ "rewards/true_env_reward_fn/std": 0.23993274569511414,
+ "step": 241,
+ "step_time": 22.238758486000734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2132116258144379,
+ "epoch": 5.902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05718924477696419,
+ "kl": 6.140609366411809e-05,
+ "learning_rate": 4.342093806672678e-07,
+ "loss": 0.012734346091747284,
+ "num_tokens": 5974626.0,
+ "reward": 0.4349059462547302,
+ "reward_std": 0.24915602803230286,
+ "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+ "rewards/true_env_reward_fn/std": 0.24915601313114166,
+ "step": 242,
+ "step_time": 9.239750460000323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 67.04167175292969,
+ "completions/mean_terminated_length": 57.574466705322266,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.7212344706058502,
+ "epoch": 5.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08872146159410477,
+ "kl": 0.00011613740025495645,
+ "learning_rate": 4.2999192751568557e-07,
+ "loss": -0.021363887935876846,
+ "num_tokens": 5997300.0,
+ "reward": 0.4436037540435791,
+ "reward_std": 0.28323379158973694,
+ "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+ "rewards/true_env_reward_fn/std": 0.28323376178741455,
+ "step": 243,
+ "step_time": 29.767976787999487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.083335876464844,
+ "completions/mean_terminated_length": 61.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2353634238243103,
+ "epoch": 5.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07658001780509949,
+ "kl": 9.086773115996039e-05,
+ "learning_rate": 4.257795488553798e-07,
+ "loss": -0.005057391710579395,
+ "num_tokens": 6021752.0,
+ "reward": 0.49971556663513184,
+ "reward_std": 0.2643933594226837,
+ "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+ "rewards/true_env_reward_fn/std": 0.2643933594226837,
+ "step": 244,
+ "step_time": 9.764708648999658
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 86.72917175292969,
+ "completions/mean_terminated_length": 77.68084716796875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2082330882549286,
+ "epoch": 5.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.059102799743413925,
+ "kl": 5.0710960749711376e-05,
+ "learning_rate": 4.215725500179787e-07,
+ "loss": 0.17436102032661438,
+ "num_tokens": 6049747.0,
+ "reward": 0.46306928992271423,
+ "reward_std": 0.26072925329208374,
+ "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+ "rewards/true_env_reward_fn/std": 0.26072925329208374,
+ "step": 245,
+ "step_time": 31.997988874000384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 61.35416793823242,
+ "completions/mean_terminated_length": 61.35416793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2454268038272858,
+ "epoch": 6.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07332625985145569,
+ "kl": 0.0001107546740968246,
+ "learning_rate": 4.1737123594515755e-07,
+ "loss": -0.013426866382360458,
+ "num_tokens": 6072668.0,
+ "reward": 0.45415109395980835,
+ "reward_std": 0.2937946617603302,
+ "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+ "rewards/true_env_reward_fn/std": 0.2937946617603302,
+ "step": 246,
+ "step_time": 9.587768273000165
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 71.5625,
+ "completions/mean_terminated_length": 71.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1738699078559875,
+ "epoch": 6.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06093747541308403,
+ "kl": 7.893411338955048e-05,
+ "learning_rate": 4.131759111665348e-07,
+ "loss": 0.022837676107883453,
+ "num_tokens": 6095263.0,
+ "reward": 0.5188159346580505,
+ "reward_std": 0.3265886902809143,
+ "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+ "rewards/true_env_reward_fn/std": 0.3265886902809143,
+ "step": 247,
+ "step_time": 14.232978527999876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3300949931144714,
+ "epoch": 6.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08098509907722473,
+ "kl": 0.00010828049380506855,
+ "learning_rate": 4.0898687977759887e-07,
+ "loss": 0.01631344109773636,
+ "num_tokens": 6125380.0,
+ "reward": 0.3338983952999115,
+ "reward_std": 0.19050820171833038,
+ "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+ "rewards/true_env_reward_fn/std": 0.19050820171833038,
+ "step": 248,
+ "step_time": 13.248441182000079
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 396.0,
+ "completions/max_terminated_length": 396.0,
+ "completions/mean_length": 88.14583587646484,
+ "completions/mean_terminated_length": 88.14583587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3161276876926422,
+ "epoch": 6.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07784765958786011,
+ "kl": 4.819030800717883e-05,
+ "learning_rate": 4.0480444541766575e-07,
+ "loss": 0.12299837917089462,
+ "num_tokens": 6151955.0,
+ "reward": 0.5446512699127197,
+ "reward_std": 0.2611033618450165,
+ "rewards/true_env_reward_fn/mean": 0.544651210308075,
+ "rewards/true_env_reward_fn/std": 0.2611033618450165,
+ "step": 249,
+ "step_time": 25.232192139999825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 69.97917175292969,
+ "completions/mean_terminated_length": 69.97917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2318329215049744,
+ "epoch": 6.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06498592346906662,
+ "kl": 3.854301621686318e-05,
+ "learning_rate": 4.0062891124787e-07,
+ "loss": -0.04302535578608513,
+ "num_tokens": 6174898.0,
+ "reward": 0.6081289052963257,
+ "reward_std": 0.24437586963176727,
+ "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+ "rewards/true_env_reward_fn/std": 0.24437588453292847,
+ "step": 250,
+ "step_time": 10.385816780999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.91666793823242,
+ "completions/mean_terminated_length": 60.91666793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2425517737865448,
+ "epoch": 6.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08402577042579651,
+ "kl": 0.00010545800250838511,
+ "learning_rate": 3.9646057992918966e-07,
+ "loss": 0.006328321993350983,
+ "num_tokens": 6203582.0,
+ "reward": 0.30250340700149536,
+ "reward_std": 0.3084425926208496,
+ "rewards/true_env_reward_fn/mean": 0.30250340700149536,
+ "rewards/true_env_reward_fn/std": 0.3084425628185272,
+ "step": 251,
+ "step_time": 9.779451584999606
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.5,
+ "completions/mean_terminated_length": 65.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2977190017700195,
+ "epoch": 6.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372989296913147,
+ "kl": 0.00017708011364447884,
+ "learning_rate": 3.9229975360050934e-07,
+ "loss": 0.04882372170686722,
+ "num_tokens": 6227486.0,
+ "reward": 0.5265982151031494,
+ "reward_std": 0.31736499071121216,
+ "rewards/true_env_reward_fn/mean": 0.5265981554985046,
+ "rewards/true_env_reward_fn/std": 0.31736496090888977,
+ "step": 252,
+ "step_time": 10.318136508000407
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 71.125,
+ "completions/mean_terminated_length": 71.125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3135383129119873,
+ "epoch": 6.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05451377481222153,
+ "kl": 6.562464477610774e-05,
+ "learning_rate": 3.8814673385671893e-07,
+ "loss": 0.01392640545964241,
+ "num_tokens": 6255436.0,
+ "reward": 0.4130678176879883,
+ "reward_std": 0.1573377549648285,
+ "rewards/true_env_reward_fn/mean": 0.4130678176879883,
+ "rewards/true_env_reward_fn/std": 0.15733776986598969,
+ "step": 253,
+ "step_time": 9.396596211999622
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 73.85417175292969,
+ "completions/mean_terminated_length": 73.85417175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3749704957008362,
+ "epoch": 6.195121951219512,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05664386227726936,
+ "kl": 6.744195252395002e-05,
+ "learning_rate": 3.840018217268526e-07,
+ "loss": -0.04205852374434471,
+ "num_tokens": 6281557.0,
+ "reward": 0.5188004970550537,
+ "reward_std": 0.2932124733924866,
+ "rewards/true_env_reward_fn/mean": 0.5188004970550537,
+ "rewards/true_env_reward_fn/std": 0.2932124435901642,
+ "step": 254,
+ "step_time": 19.77463799499992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2866049408912659,
+ "epoch": 6.219512195121951,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06639927625656128,
+ "kl": 5.235667686065426e-05,
+ "learning_rate": 3.798653176522696e-07,
+ "loss": -0.019558893516659737,
+ "num_tokens": 6302850.0,
+ "reward": 0.5025100111961365,
+ "reward_std": 0.1717289537191391,
+ "rewards/true_env_reward_fn/mean": 0.5025100111961365,
+ "rewards/true_env_reward_fn/std": 0.1717289537191391,
+ "step": 255,
+ "step_time": 9.634558264001043
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.3105897009372711,
+ "epoch": 6.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07659970223903656,
+ "kl": 8.12946273072157e-05,
+ "learning_rate": 3.7573752146487636e-07,
+ "loss": 0.05201243981719017,
+ "num_tokens": 6329269.0,
+ "reward": 0.5027985572814941,
+ "reward_std": 0.2521378695964813,
+ "rewards/true_env_reward_fn/mean": 0.5027985572814941,
+ "rewards/true_env_reward_fn/std": 0.2521378993988037,
+ "step": 256,
+ "step_time": 11.07390475200009
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.3154918253421783,
+ "epoch": 6.2682926829268295,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10612925887107849,
+ "kl": 0.00010100230429088697,
+ "learning_rate": 3.7161873236539386e-07,
+ "loss": 0.0214182510972023,
+ "num_tokens": 6356233.0,
+ "reward": 0.2817384600639343,
+ "reward_std": 0.3363598883152008,
+ "rewards/true_env_reward_fn/mean": 0.2817384600639343,
+ "rewards/true_env_reward_fn/std": 0.3363598883152008,
+ "step": 257,
+ "step_time": 10.784447634000117
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 62.770835876464844,
+ "completions/mean_terminated_length": 62.770835876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1986172497272491,
+ "epoch": 6.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08129199594259262,
+ "kl": 0.00011449725025158841,
+ "learning_rate": 3.6750924890166923e-07,
+ "loss": -0.05941678211092949,
+ "num_tokens": 6375046.0,
+ "reward": 0.5760313272476196,
+ "reward_std": 0.24430812895298004,
+ "rewards/true_env_reward_fn/mean": 0.5760312676429749,
+ "rewards/true_env_reward_fn/std": 0.24430814385414124,
+ "step": 258,
+ "step_time": 9.467202022999572
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2520000636577606,
+ "epoch": 6.317073170731708,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.06876237690448761,
+ "kl": 0.0001509120993432589,
+ "learning_rate": 3.6340936894703713e-07,
+ "loss": 0.015932239592075348,
+ "num_tokens": 6397298.0,
+ "reward": 0.4944794774055481,
+ "reward_std": 0.24476772546768188,
+ "rewards/true_env_reward_fn/mean": 0.4944794476032257,
+ "rewards/true_env_reward_fn/std": 0.24476774036884308,
+ "step": 259,
+ "step_time": 10.47640546699995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3675504922866821,
+ "epoch": 6.341463414634147,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06423573940992355,
+ "kl": 0.00010545238592385431,
+ "learning_rate": 3.593193896787277e-07,
+ "loss": 0.006066613830626011,
+ "num_tokens": 6423927.0,
+ "reward": 0.4633293151855469,
+ "reward_std": 0.2953638732433319,
+ "rewards/true_env_reward_fn/mean": 0.4633293151855469,
+ "rewards/true_env_reward_fn/std": 0.2953638732433319,
+ "step": 260,
+ "step_time": 11.672983966999709
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 74.5,
+ "completions/mean_terminated_length": 74.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3533118963241577,
+ "epoch": 6.365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07830996066331863,
+ "kl": 8.717951732251095e-05,
+ "learning_rate": 3.552396075563257e-07,
+ "loss": -0.021772567182779312,
+ "num_tokens": 6451487.0,
+ "reward": 0.3543766736984253,
+ "reward_std": 0.33776554465293884,
+ "rewards/true_env_reward_fn/mean": 0.3543766736984253,
+ "rewards/true_env_reward_fn/std": 0.33776557445526123,
+ "step": 261,
+ "step_time": 13.734938852000141
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 71.45833587646484,
+ "completions/mean_terminated_length": 71.45833587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1788119971752167,
+ "epoch": 6.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0994381383061409,
+ "kl": 0.00011037426884286106,
+ "learning_rate": 3.511703183002827e-07,
+ "loss": 0.008012240752577782,
+ "num_tokens": 6476141.0,
+ "reward": 0.45457857847213745,
+ "reward_std": 0.24262367188930511,
+ "rewards/true_env_reward_fn/mean": 0.45457854866981506,
+ "rewards/true_env_reward_fn/std": 0.24262367188930511,
+ "step": 262,
+ "step_time": 10.59573544200066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.293170690536499,
+ "epoch": 6.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10550480335950851,
+ "kl": 0.00020241059792169835,
+ "learning_rate": 3.4711181687048106e-07,
+ "loss": 0.02680305391550064,
+ "num_tokens": 6499670.0,
+ "reward": 0.33261698484420776,
+ "reward_std": 0.31543657183647156,
+ "rewards/true_env_reward_fn/mean": 0.3326169550418854,
+ "rewards/true_env_reward_fn/std": 0.31543657183647156,
+ "step": 263,
+ "step_time": 10.02452396199942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 89.83333587646484,
+ "completions/mean_terminated_length": 89.83333587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2982321977615356,
+ "epoch": 6.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06412192434072495,
+ "kl": 6.536830551340245e-05,
+ "learning_rate": 3.4306439744485447e-07,
+ "loss": -0.08396965265274048,
+ "num_tokens": 6522630.0,
+ "reward": 0.5518762469291687,
+ "reward_std": 0.21314994990825653,
+ "rewards/true_env_reward_fn/mean": 0.5518762469291687,
+ "rewards/true_env_reward_fn/std": 0.21314994990825653,
+ "step": 264,
+ "step_time": 20.82168071200067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 63.85416793823242,
+ "completions/mean_terminated_length": 63.85416793823242,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2279660403728485,
+ "epoch": 6.463414634146342,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10119301080703735,
+ "kl": 0.00010847221983567579,
+ "learning_rate": 3.3902835339806456e-07,
+ "loss": 0.01589711755514145,
+ "num_tokens": 6548183.0,
+ "reward": 0.4527897238731384,
+ "reward_std": 0.2534019947052002,
+ "rewards/true_env_reward_fn/mean": 0.45278969407081604,
+ "rewards/true_env_reward_fn/std": 0.2534019649028778,
+ "step": 265,
+ "step_time": 8.63894235699945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 60.54166793823242,
+ "completions/mean_terminated_length": 60.54166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1612891554832458,
+ "epoch": 6.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0825633853673935,
+ "kl": 7.345602170971688e-05,
+ "learning_rate": 3.3500397728023534e-07,
+ "loss": 0.057398565113544464,
+ "num_tokens": 6571537.0,
+ "reward": 0.5196421146392822,
+ "reward_std": 0.18547315895557404,
+ "rewards/true_env_reward_fn/mean": 0.5196421146392822,
+ "rewards/true_env_reward_fn/std": 0.18547315895557404,
+ "step": 266,
+ "step_time": 10.01155260700034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2408597469329834,
+ "epoch": 6.512195121951219,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09217255562543869,
+ "kl": 9.46905902310391e-05,
+ "learning_rate": 3.3099156079574867e-07,
+ "loss": 0.026750221848487854,
+ "num_tokens": 6596010.0,
+ "reward": 0.4330406188964844,
+ "reward_std": 0.20423907041549683,
+ "rewards/true_env_reward_fn/mean": 0.4330406188964844,
+ "rewards/true_env_reward_fn/std": 0.20423908531665802,
+ "step": 267,
+ "step_time": 20.002466699999786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 64.83333587646484,
+ "completions/mean_terminated_length": 64.83333587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2907497882843018,
+ "epoch": 6.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07903403043746948,
+ "kl": 0.00015101409735507332,
+ "learning_rate": 3.269913947820998e-07,
+ "loss": 0.02006390690803528,
+ "num_tokens": 6622106.0,
+ "reward": 0.4544076919555664,
+ "reward_std": 0.26717478036880493,
+ "rewards/true_env_reward_fn/mean": 0.4544076919555664,
+ "rewards/true_env_reward_fn/std": 0.2671748101711273,
+ "step": 268,
+ "step_time": 8.818348709999555
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2150432765483856,
+ "epoch": 6.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09289928525686264,
+ "kl": 9.474463149672374e-05,
+ "learning_rate": 3.230037691888162e-07,
+ "loss": 0.08772514015436172,
+ "num_tokens": 6636101.0,
+ "reward": 0.6059033870697021,
+ "reward_std": 0.23812197148799896,
+ "rewards/true_env_reward_fn/mean": 0.6059033274650574,
+ "rewards/true_env_reward_fn/std": 0.23812197148799896,
+ "step": 269,
+ "step_time": 8.47666211500109
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.2745259404182434,
+ "epoch": 6.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09368392825126648,
+ "kl": 0.0001518711451353738,
+ "learning_rate": 3.1902897305644093e-07,
+ "loss": -0.003839995712041855,
+ "num_tokens": 6659877.0,
+ "reward": 0.5029901266098022,
+ "reward_std": 0.3467065095901489,
+ "rewards/true_env_reward_fn/mean": 0.5029900670051575,
+ "rewards/true_env_reward_fn/std": 0.3467065393924713,
+ "step": 270,
+ "step_time": 11.298448464000103
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 67.0625,
+ "completions/mean_terminated_length": 67.0625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.238816499710083,
+ "epoch": 6.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05819084122776985,
+ "kl": 6.655609695371822e-05,
+ "learning_rate": 3.150672944955818e-07,
+ "loss": -0.02771003544330597,
+ "num_tokens": 6679776.0,
+ "reward": 0.5772091746330261,
+ "reward_std": 0.17815756797790527,
+ "rewards/true_env_reward_fn/mean": 0.5772091746330261,
+ "rewards/true_env_reward_fn/std": 0.17815756797790527,
+ "step": 271,
+ "step_time": 7.902968623000561
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 145.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 75.10417175292969,
+ "completions/mean_terminated_length": 75.10417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3645851016044617,
+ "epoch": 6.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07474905252456665,
+ "kl": 7.497054593841312e-05,
+ "learning_rate": 3.1111902066602724e-07,
+ "loss": 0.07271203398704529,
+ "num_tokens": 6704717.0,
+ "reward": 0.42504438757896423,
+ "reward_std": 0.2500284016132355,
+ "rewards/true_env_reward_fn/mean": 0.42504438757896423,
+ "rewards/true_env_reward_fn/std": 0.2500284016132355,
+ "step": 272,
+ "step_time": 11.259095110999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 67.64583587646484,
+ "completions/mean_terminated_length": 67.64583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2410458326339722,
+ "epoch": 6.658536585365853,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0750170573592186,
+ "kl": 8.587932643422391e-05,
+ "learning_rate": 3.0718443775593225e-07,
+ "loss": -0.019169267266988754,
+ "num_tokens": 6727060.0,
+ "reward": 0.5114523768424988,
+ "reward_std": 0.17165428400039673,
+ "rewards/true_env_reward_fn/mean": 0.5114523768424988,
+ "rewards/true_env_reward_fn/std": 0.17165428400039673,
+ "step": 273,
+ "step_time": 11.097374408000178
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 67.08333587646484,
+ "completions/mean_terminated_length": 67.08333587646484,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2417791783809662,
+ "epoch": 6.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08969064801931381,
+ "kl": 5.61167857995315e-05,
+ "learning_rate": 3.0326383096107423e-07,
+ "loss": 0.11341358721256256,
+ "num_tokens": 6752824.0,
+ "reward": 0.4772833287715912,
+ "reward_std": 0.31618404388427734,
+ "rewards/true_env_reward_fn/mean": 0.4772833287715912,
+ "rewards/true_env_reward_fn/std": 0.31618407368659973,
+ "step": 274,
+ "step_time": 11.189890726001067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.47917175292969,
+ "completions/mean_terminated_length": 67.47917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.272821843624115,
+ "epoch": 6.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08874963968992233,
+ "kl": 0.00015055539006425533,
+ "learning_rate": 2.9935748446418065e-07,
+ "loss": 0.02028803899884224,
+ "num_tokens": 6776703.0,
+ "reward": 0.49093031883239746,
+ "reward_std": 0.25163188576698303,
+ "rewards/true_env_reward_fn/mean": 0.49093031883239746,
+ "rewards/true_env_reward_fn/std": 0.25163188576698303,
+ "step": 275,
+ "step_time": 9.941926390000845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 365.0,
+ "completions/max_terminated_length": 365.0,
+ "completions/mean_length": 68.04167175292969,
+ "completions/mean_terminated_length": 68.04167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0912510752677917,
+ "epoch": 6.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060312703251838684,
+ "kl": 6.840269179519964e-05,
+ "learning_rate": 2.9546568141433e-07,
+ "loss": -0.036469291895627975,
+ "num_tokens": 6799417.0,
+ "reward": 0.5543485879898071,
+ "reward_std": 0.22147472202777863,
+ "rewards/true_env_reward_fn/mean": 0.5543485283851624,
+ "rewards/true_env_reward_fn/std": 0.22147469222545624,
+ "step": 276,
+ "step_time": 22.291117544999906
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.895835876464844,
+ "completions/mean_terminated_length": 59.895835876464844,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.2566059231758118,
+ "epoch": 6.7560975609756095,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09982399642467499,
+ "kl": 0.00020091429905733094,
+ "learning_rate": 2.9158870390642863e-07,
+ "loss": -0.005738064646720886,
+ "num_tokens": 6829804.0,
+ "reward": 0.24754562973976135,
+ "reward_std": 0.23038579523563385,
+ "rewards/true_env_reward_fn/mean": 0.24754561483860016,
+ "rewards/true_env_reward_fn/std": 0.23038578033447266,
+ "step": 277,
+ "step_time": 9.732460060999983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 149.0,
+ "completions/max_terminated_length": 149.0,
+ "completions/mean_length": 71.60417175292969,
+ "completions/mean_terminated_length": 71.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2196767628192902,
+ "epoch": 6.780487804878049,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03178449347615242,
+ "kl": 5.627466816804372e-05,
+ "learning_rate": 2.8772683296076194e-07,
+ "loss": 0.002586783841252327,
+ "num_tokens": 6852617.0,
+ "reward": 0.5281299352645874,
+ "reward_std": 0.22268518805503845,
+ "rewards/true_env_reward_fn/mean": 0.5281298756599426,
+ "rewards/true_env_reward_fn/std": 0.22268518805503845,
+ "step": 278,
+ "step_time": 11.629210506000163
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 169.0,
+ "completions/max_terminated_length": 169.0,
+ "completions/mean_length": 68.66667175292969,
+ "completions/mean_terminated_length": 68.66667175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1991542279720306,
+ "epoch": 6.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07567304372787476,
+ "kl": 0.00015181046910583973,
+ "learning_rate": 2.8388034850262646e-07,
+ "loss": -0.04906900227069855,
+ "num_tokens": 6881529.0,
+ "reward": 0.3906375765800476,
+ "reward_std": 0.3576822578907013,
+ "rewards/true_env_reward_fn/mean": 0.3906375467777252,
+ "rewards/true_env_reward_fn/std": 0.3576822876930237,
+ "step": 279,
+ "step_time": 18.253660386000774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 208.0,
+ "completions/max_terminated_length": 208.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.263102412223816,
+ "epoch": 6.829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09104589372873306,
+ "kl": 0.00012209633769089123,
+ "learning_rate": 2.8004952934203837e-07,
+ "loss": 0.06642289459705353,
+ "num_tokens": 6905111.0,
+ "reward": 0.3988339304924011,
+ "reward_std": 0.34396088123321533,
+ "rewards/true_env_reward_fn/mean": 0.39883390069007874,
+ "rewards/true_env_reward_fn/std": 0.3439609110355377,
+ "step": 280,
+ "step_time": 15.519001798998943
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 59.0625,
+ "completions/mean_terminated_length": 59.0625,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.208267629146576,
+ "epoch": 6.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07465670257806778,
+ "kl": 7.14320785846212e-05,
+ "learning_rate": 2.762346531535246e-07,
+ "loss": 0.04124641790986061,
+ "num_tokens": 6926634.0,
+ "reward": 0.5870868563652039,
+ "reward_std": 0.28217118978500366,
+ "rewards/true_env_reward_fn/mean": 0.5870868563652039,
+ "rewards/true_env_reward_fn/std": 0.2821711599826813,
+ "step": 281,
+ "step_time": 11.224198447001072
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.02083587646484,
+ "completions/mean_terminated_length": 72.02083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3142951428890228,
+ "epoch": 6.878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07364725321531296,
+ "kl": 0.00011016946336894762,
+ "learning_rate": 2.7243599645599574e-07,
+ "loss": 0.004198473412543535,
+ "num_tokens": 6960219.0,
+ "reward": 0.34401482343673706,
+ "reward_std": 0.24699951708316803,
+ "rewards/true_env_reward_fn/mean": 0.3440147936344147,
+ "rewards/true_env_reward_fn/std": 0.24699951708316803,
+ "step": 282,
+ "step_time": 15.210776117999558
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 60.583335876464844,
+ "completions/mean_terminated_length": 60.583335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2013934552669525,
+ "epoch": 6.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07040710747241974,
+ "kl": 7.988750212462037e-05,
+ "learning_rate": 2.686538345927026e-07,
+ "loss": 0.06334929168224335,
+ "num_tokens": 6982879.0,
+ "reward": 0.5139331221580505,
+ "reward_std": 0.19786590337753296,
+ "rewards/true_env_reward_fn/mean": 0.5139331221580505,
+ "rewards/true_env_reward_fn/std": 0.19786591827869415,
+ "step": 283,
+ "step_time": 10.240000448000501
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2223551571369171,
+ "epoch": 6.926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07971613109111786,
+ "kl": 0.00016717875041649677,
+ "learning_rate": 2.64888441711279e-07,
+ "loss": 0.03706979751586914,
+ "num_tokens": 7003990.0,
+ "reward": 0.5301157832145691,
+ "reward_std": 0.2464885115623474,
+ "rewards/true_env_reward_fn/mean": 0.5301157832145691,
+ "rewards/true_env_reward_fn/std": 0.2464885115623474,
+ "step": 284,
+ "step_time": 9.309556909001913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.309690535068512,
+ "epoch": 6.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09541535377502441,
+ "kl": 0.00014337312313728034,
+ "learning_rate": 2.6114009074386844e-07,
+ "loss": -0.04744558781385422,
+ "num_tokens": 7027733.0,
+ "reward": 0.468932569026947,
+ "reward_std": 0.2396899163722992,
+ "rewards/true_env_reward_fn/mean": 0.46893253922462463,
+ "rewards/true_env_reward_fn/std": 0.2396899312734604,
+ "step": 285,
+ "step_time": 8.91306197799986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1709823310375214,
+ "epoch": 6.975609756097561,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048187412321567535,
+ "kl": 9.718035107653122e-05,
+ "learning_rate": 2.5740905338734306e-07,
+ "loss": 0.010684527456760406,
+ "num_tokens": 7058137.0,
+ "reward": 0.44316989183425903,
+ "reward_std": 0.14355739951133728,
+ "rewards/true_env_reward_fn/mean": 0.44316986203193665,
+ "rewards/true_env_reward_fn/std": 0.14355739951133728,
+ "step": 286,
+ "step_time": 9.546786461999545
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 458.0,
+ "completions/max_terminated_length": 458.0,
+ "completions/mean_length": 77.97917175292969,
+ "completions/mean_terminated_length": 77.97917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3698437809944153,
+ "epoch": 7.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05576475337147713,
+ "kl": 9.964485252567101e-05,
+ "learning_rate": 2.536956000836082e-07,
+ "loss": 0.08634226769208908,
+ "num_tokens": 7082840.0,
+ "reward": 0.5030691027641296,
+ "reward_std": 0.2275215983390808,
+ "rewards/true_env_reward_fn/mean": 0.5030691027641296,
+ "rewards/true_env_reward_fn/std": 0.22752158343791962,
+ "step": 287,
+ "step_time": 26.703723129000537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2532718479633331,
+ "epoch": 7.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08579511940479279,
+ "kl": 9.319775290350663e-05,
+ "learning_rate": 2.500000000000001e-07,
+ "loss": -0.013113420456647873,
+ "num_tokens": 7098689.0,
+ "reward": 0.5958684682846069,
+ "reward_std": 0.1863170713186264,
+ "rewards/true_env_reward_fn/mean": 0.5958684682846069,
+ "rewards/true_env_reward_fn/std": 0.1863170564174652,
+ "step": 288,
+ "step_time": 7.618657231000725
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.20833587646484,
+ "completions/mean_terminated_length": 68.20833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.1685407161712646,
+ "epoch": 7.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07076086848974228,
+ "kl": 7.435419411194744e-05,
+ "learning_rate": 2.4632252100977564e-07,
+ "loss": 0.010196422226727009,
+ "num_tokens": 7126387.0,
+ "reward": 0.47137731313705444,
+ "reward_std": 0.20232117176055908,
+ "rewards/true_env_reward_fn/mean": 0.47137728333473206,
+ "rewards/true_env_reward_fn/std": 0.20232117176055908,
+ "step": 289,
+ "step_time": 10.741382757999418
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2131870985031128,
+ "epoch": 7.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07972414791584015,
+ "kl": 0.00010809541072376305,
+ "learning_rate": 2.426634296726955e-07,
+ "loss": 0.07707639783620834,
+ "num_tokens": 7150947.0,
+ "reward": 0.4951697587966919,
+ "reward_std": 0.31705158948898315,
+ "rewards/true_env_reward_fn/mean": 0.4951697587966919,
+ "rewards/true_env_reward_fn/std": 0.31705158948898315,
+ "step": 290,
+ "step_time": 11.908707627000695
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 66.41667175292969,
+ "completions/mean_terminated_length": 66.41667175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2247934639453888,
+ "epoch": 7.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06208934634923935,
+ "kl": 8.92497146196547e-05,
+ "learning_rate": 2.390229912157033e-07,
+ "loss": 0.021278446540236473,
+ "num_tokens": 7174671.0,
+ "reward": 0.44533461332321167,
+ "reward_std": 0.20755091309547424,
+ "rewards/true_env_reward_fn/mean": 0.4453345835208893,
+ "rewards/true_env_reward_fn/std": 0.20755092799663544,
+ "step": 291,
+ "step_time": 11.859711304000484
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.25,
+ "completions/mean_terminated_length": 66.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3344177305698395,
+ "epoch": 7.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08249509334564209,
+ "kl": 0.00013599474277725676,
+ "learning_rate": 2.3540146951369966e-07,
+ "loss": 0.05746981129050255,
+ "num_tokens": 7202291.0,
+ "reward": 0.36803489923477173,
+ "reward_std": 0.22159868478775024,
+ "rewards/true_env_reward_fn/mean": 0.36803486943244934,
+ "rewards/true_env_reward_fn/std": 0.22159868478775024,
+ "step": 292,
+ "step_time": 9.794866193998132
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 64.02083587646484,
+ "completions/mean_terminated_length": 64.02083587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2605153024196625,
+ "epoch": 7.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07226194441318512,
+ "kl": 7.86567807153915e-05,
+ "learning_rate": 2.3179912707041666e-07,
+ "loss": -0.06701754778623581,
+ "num_tokens": 7221060.0,
+ "reward": 0.6259939670562744,
+ "reward_std": 0.30331701040267944,
+ "rewards/true_env_reward_fn/mean": 0.6259939074516296,
+ "rewards/true_env_reward_fn/std": 0.30331701040267944,
+ "step": 293,
+ "step_time": 13.536596455999643
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 76.83333587646484,
+ "completions/mean_terminated_length": 76.83333587646484,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.275952398777008,
+ "epoch": 7.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07967559248209,
+ "kl": 0.0001376894815621199,
+ "learning_rate": 2.2821622499938948e-07,
+ "loss": -0.024285754188895226,
+ "num_tokens": 7251156.0,
+ "reward": 0.3353421688079834,
+ "reward_std": 0.26816248893737793,
+ "rewards/true_env_reward_fn/mean": 0.3353421688079834,
+ "rewards/true_env_reward_fn/std": 0.26816248893737793,
+ "step": 294,
+ "step_time": 19.762229363001097
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.4140338003635406,
+ "epoch": 7.195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07337357848882675,
+ "kl": 0.00012654263991862535,
+ "learning_rate": 2.2465302300503008e-07,
+ "loss": -0.04723845049738884,
+ "num_tokens": 7277361.0,
+ "reward": 0.4155184030532837,
+ "reward_std": 0.23990045487880707,
+ "rewards/true_env_reward_fn/mean": 0.4155184030532837,
+ "rewards/true_env_reward_fn/std": 0.23990046977996826,
+ "step": 295,
+ "step_time": 11.263231479000751
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 63.9375,
+ "completions/mean_terminated_length": 63.9375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.22734934091568,
+ "epoch": 7.219512195121951,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0908711850643158,
+ "kl": 0.00016623977535346057,
+ "learning_rate": 2.2110977936380287e-07,
+ "loss": -0.03445049002766609,
+ "num_tokens": 7298998.0,
+ "reward": 0.564518392086029,
+ "reward_std": 0.30392351746559143,
+ "rewards/true_env_reward_fn/mean": 0.564518392086029,
+ "rewards/true_env_reward_fn/std": 0.30392348766326904,
+ "step": 296,
+ "step_time": 10.9702629049998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.312496393918991,
+ "epoch": 7.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08918203413486481,
+ "kl": 0.00018221777827420738,
+ "learning_rate": 2.1758675090550328e-07,
+ "loss": 0.023428799584507942,
+ "num_tokens": 7320725.0,
+ "reward": 0.4695018231868744,
+ "reward_std": 0.2527172565460205,
+ "rewards/true_env_reward_fn/mean": 0.4695018231868744,
+ "rewards/true_env_reward_fn/std": 0.2527172565460205,
+ "step": 297,
+ "step_time": 12.689384352000161
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.260593295097351,
+ "epoch": 7.2682926829268295,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07724963873624802,
+ "kl": 4.775456409333856e-05,
+ "learning_rate": 2.1408419299464242e-07,
+ "loss": 0.03472111374139786,
+ "num_tokens": 7345397.0,
+ "reward": 0.39238378405570984,
+ "reward_std": 0.23648974299430847,
+ "rewards/true_env_reward_fn/mean": 0.39238378405570984,
+ "rewards/true_env_reward_fn/std": 0.23648977279663086,
+ "step": 298,
+ "step_time": 10.398283558999537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.272550791501999,
+ "epoch": 7.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0737687349319458,
+ "kl": 7.711273838140187e-05,
+ "learning_rate": 2.1060235951193578e-07,
+ "loss": 0.010874807834625244,
+ "num_tokens": 7371231.0,
+ "reward": 0.4486481845378876,
+ "reward_std": 0.26590272784233093,
+ "rewards/true_env_reward_fn/mean": 0.4486481845378876,
+ "rewards/true_env_reward_fn/std": 0.26590269804000854,
+ "step": 299,
+ "step_time": 12.961759718001304
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3426357805728912,
+ "epoch": 7.317073170731708,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09169661998748779,
+ "kl": 0.00015296797573682852,
+ "learning_rate": 2.071415028359026e-07,
+ "loss": -0.0708877444267273,
+ "num_tokens": 7397151.0,
+ "reward": 0.5012298822402954,
+ "reward_std": 0.21363919973373413,
+ "rewards/true_env_reward_fn/mean": 0.5012298226356506,
+ "rewards/true_env_reward_fn/std": 0.21363921463489532,
+ "step": 300,
+ "step_time": 13.761256955000135
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.75,
+ "completions/mean_terminated_length": 70.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2883423566818237,
+ "epoch": 7.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06652592122554779,
+ "kl": 0.00011371383516234346,
+ "learning_rate": 2.0370187382457066e-07,
+ "loss": -0.011642830446362495,
+ "num_tokens": 7426403.0,
+ "reward": 0.3339906632900238,
+ "reward_std": 0.20955638587474823,
+ "rewards/true_env_reward_fn/mean": 0.3339906632900238,
+ "rewards/true_env_reward_fn/std": 0.20955640077590942,
+ "step": 301,
+ "step_time": 9.628323140000248
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 68.02083587646484,
+ "completions/mean_terminated_length": 68.02083587646484,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.1677636206150055,
+ "epoch": 7.365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08921164274215698,
+ "kl": 7.750577606202569e-05,
+ "learning_rate": 2.0028372179729402e-07,
+ "loss": 0.024114318192005157,
+ "num_tokens": 7458380.0,
+ "reward": 0.3653101921081543,
+ "reward_std": 0.3042241334915161,
+ "rewards/true_env_reward_fn/mean": 0.3653101921081543,
+ "rewards/true_env_reward_fn/std": 0.3042241334915161,
+ "step": 302,
+ "step_time": 17.152215452000746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.04166793823242,
+ "completions/mean_terminated_length": 61.04166793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1981053352355957,
+ "epoch": 7.390243902439025,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08189807087182999,
+ "kl": 0.00016033334395615384,
+ "learning_rate": 1.9688729451668111e-07,
+ "loss": 0.017046045511960983,
+ "num_tokens": 7481590.0,
+ "reward": 0.5003601312637329,
+ "reward_std": 0.3917968273162842,
+ "rewards/true_env_reward_fn/mean": 0.5003601312637329,
+ "rewards/true_env_reward_fn/std": 0.3917968273162842,
+ "step": 303,
+ "step_time": 10.428820308000468
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 64.95833587646484,
+ "completions/mean_terminated_length": 64.95833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2136133313179016,
+ "epoch": 7.414634146341464,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07148941606283188,
+ "kl": 5.305510785547085e-05,
+ "learning_rate": 1.9351283817063546e-07,
+ "loss": -0.009052902460098267,
+ "num_tokens": 7508932.0,
+ "reward": 0.3940971791744232,
+ "reward_std": 0.2546152174472809,
+ "rewards/true_env_reward_fn/mean": 0.3940971791744232,
+ "rewards/true_env_reward_fn/std": 0.25461524724960327,
+ "step": 304,
+ "step_time": 12.12407543900099
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2293521761894226,
+ "epoch": 7.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05877150595188141,
+ "kl": 0.00012415168203006033,
+ "learning_rate": 1.9016059735451156e-07,
+ "loss": -0.008430279791355133,
+ "num_tokens": 7535518.0,
+ "reward": 0.47240138053894043,
+ "reward_std": 0.2845028340816498,
+ "rewards/true_env_reward_fn/mean": 0.47240138053894043,
+ "rewards/true_env_reward_fn/std": 0.2845028340816498,
+ "step": 305,
+ "step_time": 12.694503639000686
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1673714816570282,
+ "epoch": 7.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07308074831962585,
+ "kl": 0.0001147145967479446,
+ "learning_rate": 1.8683081505338465e-07,
+ "loss": -0.06511729210615158,
+ "num_tokens": 7558883.0,
+ "reward": 0.4651026129722595,
+ "reward_std": 0.19122423231601715,
+ "rewards/true_env_reward_fn/mean": 0.46510258316993713,
+ "rewards/true_env_reward_fn/std": 0.19122423231601715,
+ "step": 306,
+ "step_time": 15.502204728000834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 63.6875,
+ "completions/mean_terminated_length": 63.6875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.304062157869339,
+ "epoch": 7.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0670589879155159,
+ "kl": 9.212431177729741e-05,
+ "learning_rate": 1.8352373262443915e-07,
+ "loss": -0.026926759630441666,
+ "num_tokens": 7578916.0,
+ "reward": 0.5716189742088318,
+ "reward_std": 0.19298586249351501,
+ "rewards/true_env_reward_fn/mean": 0.5716189742088318,
+ "rewards/true_env_reward_fn/std": 0.19298586249351501,
+ "step": 307,
+ "step_time": 8.911180752998916
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2283632457256317,
+ "epoch": 7.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06950085610151291,
+ "kl": 9.545813054501195e-05,
+ "learning_rate": 1.80239589779473e-07,
+ "loss": 0.010137543082237244,
+ "num_tokens": 7604028.0,
+ "reward": 0.5349916219711304,
+ "reward_std": 0.2091754674911499,
+ "rewards/true_env_reward_fn/mean": 0.5349915623664856,
+ "rewards/true_env_reward_fn/std": 0.2091754525899887,
+ "step": 308,
+ "step_time": 11.745030509000571
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 77.5,
+ "completions/mean_terminated_length": 77.5,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3315171897411346,
+ "epoch": 7.536585365853659,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05699850618839264,
+ "kl": 6.70248282403918e-05,
+ "learning_rate": 1.7697862456752271e-07,
+ "loss": 0.06499005854129791,
+ "num_tokens": 7628884.0,
+ "reward": 0.4583873748779297,
+ "reward_std": 0.26091766357421875,
+ "rewards/true_env_reward_fn/mean": 0.4583873748779297,
+ "rewards/true_env_reward_fn/std": 0.26091763377189636,
+ "step": 309,
+ "step_time": 13.87453935099984
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2704036831855774,
+ "epoch": 7.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09379290789365768,
+ "kl": 0.0001983325491892174,
+ "learning_rate": 1.7374107335760934e-07,
+ "loss": 0.09065254032611847,
+ "num_tokens": 7651991.0,
+ "reward": 0.5210780501365662,
+ "reward_std": 0.2745552361011505,
+ "rewards/true_env_reward_fn/mean": 0.5210780501365662,
+ "rewards/true_env_reward_fn/std": 0.2745552361011505,
+ "step": 310,
+ "step_time": 11.07200519900016
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.60417175292969,
+ "completions/mean_terminated_length": 73.60417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2427658140659332,
+ "epoch": 7.585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06011087819933891,
+ "kl": 6.545234282384627e-05,
+ "learning_rate": 1.7052717082160344e-07,
+ "loss": -0.0115435142070055,
+ "num_tokens": 7682948.0,
+ "reward": 0.3796111047267914,
+ "reward_std": 0.2212861329317093,
+ "rewards/true_env_reward_fn/mean": 0.3796111047267914,
+ "rewards/true_env_reward_fn/std": 0.22128616273403168,
+ "step": 311,
+ "step_time": 12.894371897999918
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1208404004573822,
+ "epoch": 7.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0799943059682846,
+ "kl": 0.0001426433700544294,
+ "learning_rate": 1.6733714991721738e-07,
+ "loss": -0.010216867551207542,
+ "num_tokens": 7704179.0,
+ "reward": 0.5524939298629761,
+ "reward_std": 0.26985135674476624,
+ "rewards/true_env_reward_fn/mean": 0.5524939298629761,
+ "rewards/true_env_reward_fn/std": 0.26985135674476624,
+ "step": 312,
+ "step_time": 8.69524126000033
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2062768340110779,
+ "epoch": 7.634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05652271583676338,
+ "kl": 0.00010435856393087306,
+ "learning_rate": 1.6417124187111774e-07,
+ "loss": 0.033915065228939056,
+ "num_tokens": 7732974.0,
+ "reward": 0.40553492307662964,
+ "reward_std": 0.2561950087547302,
+ "rewards/true_env_reward_fn/mean": 0.40553489327430725,
+ "rewards/true_env_reward_fn/std": 0.2561950087547302,
+ "step": 313,
+ "step_time": 11.456125995000548
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 63.41666793823242,
+ "completions/mean_terminated_length": 63.41666793823242,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2864463925361633,
+ "epoch": 7.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08890142291784286,
+ "kl": 0.00014215287774277385,
+ "learning_rate": 1.6102967616216617e-07,
+ "loss": 0.04210362955927849,
+ "num_tokens": 7754650.0,
+ "reward": 0.5954334139823914,
+ "reward_std": 0.3237621784210205,
+ "rewards/true_env_reward_fn/mean": 0.5954334139823914,
+ "rewards/true_env_reward_fn/std": 0.3237621784210205,
+ "step": 314,
+ "step_time": 11.561733381999147
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 69.22917175292969,
+ "completions/mean_terminated_length": 69.22917175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2406023740768433,
+ "epoch": 7.682926829268292,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05483023077249527,
+ "kl": 3.3986466860369546e-05,
+ "learning_rate": 1.5791268050478483e-07,
+ "loss": -0.03052404522895813,
+ "num_tokens": 7777389.0,
+ "reward": 0.5392192602157593,
+ "reward_std": 0.21974749863147736,
+ "rewards/true_env_reward_fn/mean": 0.5392192006111145,
+ "rewards/true_env_reward_fn/std": 0.21974751353263855,
+ "step": 315,
+ "step_time": 13.48241268899983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2167797982692719,
+ "epoch": 7.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08746550232172012,
+ "kl": 0.00011399560207792092,
+ "learning_rate": 1.5482048083245114e-07,
+ "loss": 0.014903642237186432,
+ "num_tokens": 7807828.0,
+ "reward": 0.29947829246520996,
+ "reward_std": 0.2547810673713684,
+ "rewards/true_env_reward_fn/mean": 0.29947829246520996,
+ "rewards/true_env_reward_fn/std": 0.2547810673713684,
+ "step": 316,
+ "step_time": 10.080044547000398
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 65.3125,
+ "completions/mean_terminated_length": 65.3125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2823624312877655,
+ "epoch": 7.7317073170731705,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07846319675445557,
+ "kl": 0.00012852110194216948,
+ "learning_rate": 1.517533012813217e-07,
+ "loss": 0.05300665646791458,
+ "num_tokens": 7834227.0,
+ "reward": 0.4816446304321289,
+ "reward_std": 0.3394080102443695,
+ "rewards/true_env_reward_fn/mean": 0.4816446304321289,
+ "rewards/true_env_reward_fn/std": 0.3394079804420471,
+ "step": 317,
+ "step_time": 12.856840839001052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 264.0,
+ "completions/max_terminated_length": 264.0,
+ "completions/mean_length": 83.47917175292969,
+ "completions/mean_terminated_length": 83.47917175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3714390099048615,
+ "epoch": 7.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0722746029496193,
+ "kl": 0.00011485655522847082,
+ "learning_rate": 1.4871136417398405e-07,
+ "loss": -0.009422918781638145,
+ "num_tokens": 7860362.0,
+ "reward": 0.44833892583847046,
+ "reward_std": 0.24655339121818542,
+ "rewards/true_env_reward_fn/mean": 0.44833889603614807,
+ "rewards/true_env_reward_fn/std": 0.24655337631702423,
+ "step": 318,
+ "step_time": 18.09142264499951
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2680339813232422,
+ "epoch": 7.780487804878049,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09518000483512878,
+ "kl": 0.00010953140827041352,
+ "learning_rate": 1.4569489000334433e-07,
+ "loss": -0.09500816464424133,
+ "num_tokens": 7882799.0,
+ "reward": 0.4905685782432556,
+ "reward_std": 0.261008620262146,
+ "rewards/true_env_reward_fn/mean": 0.4905685484409332,
+ "rewards/true_env_reward_fn/std": 0.261008620262146,
+ "step": 319,
+ "step_time": 17.2091521250004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 257.0,
+ "completions/max_terminated_length": 257.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2107920348644257,
+ "epoch": 7.804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07063736766576767,
+ "kl": 3.921870529666194e-05,
+ "learning_rate": 1.4270409741664268e-07,
+ "loss": -0.011502111330628395,
+ "num_tokens": 7901717.0,
+ "reward": 0.566825807094574,
+ "reward_std": 0.18909600377082825,
+ "rewards/true_env_reward_fn/mean": 0.566825807094574,
+ "rewards/true_env_reward_fn/std": 0.18909598886966705,
+ "step": 320,
+ "step_time": 15.649325063000106
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 66.66667175292969,
+ "completions/mean_terminated_length": 66.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2565636038780212,
+ "epoch": 7.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06063119322061539,
+ "kl": 4.300068394513801e-05,
+ "learning_rate": 1.3973920319960652e-07,
+ "loss": 0.001966973766684532,
+ "num_tokens": 7919013.0,
+ "reward": 0.6115278005599976,
+ "reward_std": 0.17882205545902252,
+ "rewards/true_env_reward_fn/mean": 0.6115277409553528,
+ "rewards/true_env_reward_fn/std": 0.17882204055786133,
+ "step": 321,
+ "step_time": 7.359487544999865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 57.083335876464844,
+ "completions/mean_terminated_length": 57.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2200327813625336,
+ "epoch": 7.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07409472018480301,
+ "kl": 0.00011575021017051768,
+ "learning_rate": 1.368004222607355e-07,
+ "loss": -0.047135986387729645,
+ "num_tokens": 7941137.0,
+ "reward": 0.5057475566864014,
+ "reward_std": 0.3366250991821289,
+ "rewards/true_env_reward_fn/mean": 0.5057475566864014,
+ "rewards/true_env_reward_fn/std": 0.3366251289844513,
+ "step": 322,
+ "step_time": 10.691665401999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 57.520835876464844,
+ "completions/mean_terminated_length": 57.520835876464844,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.3097383677959442,
+ "epoch": 7.878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0950520932674408,
+ "kl": 0.00014910039135429543,
+ "learning_rate": 1.338879676157249e-07,
+ "loss": -0.018300604075193405,
+ "num_tokens": 7967298.0,
+ "reward": 0.39754772186279297,
+ "reward_std": 0.227029949426651,
+ "rewards/true_env_reward_fn/mean": 0.39754772186279297,
+ "rewards/true_env_reward_fn/std": 0.2270299643278122,
+ "step": 323,
+ "step_time": 9.646710404997975
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 165.0,
+ "completions/max_terminated_length": 165.0,
+ "completions/mean_length": 63.04166793823242,
+ "completions/mean_terminated_length": 63.04166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3998334109783173,
+ "epoch": 7.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08187483251094818,
+ "kl": 0.0001343226194876479,
+ "learning_rate": 1.310020503720254e-07,
+ "loss": 0.10223554074764252,
+ "num_tokens": 7989452.0,
+ "reward": 0.4514659643173218,
+ "reward_std": 0.3259531557559967,
+ "rewards/true_env_reward_fn/mean": 0.4514659643173218,
+ "rewards/true_env_reward_fn/std": 0.3259531557559967,
+ "step": 324,
+ "step_time": 13.284335969000495
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 78.77083587646484,
+ "completions/mean_terminated_length": 78.77083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3267191052436829,
+ "epoch": 7.926829268292683,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06648680567741394,
+ "kl": 5.966442222415935e-05,
+ "learning_rate": 1.281428797135402e-07,
+ "loss": -0.046651843935251236,
+ "num_tokens": 8011065.0,
+ "reward": 0.6008384227752686,
+ "reward_std": 0.19184507429599762,
+ "rewards/true_env_reward_fn/mean": 0.6008384227752686,
+ "rewards/true_env_reward_fn/std": 0.19184507429599762,
+ "step": 325,
+ "step_time": 18.06017050799983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 61.54166793823242,
+ "completions/mean_terminated_length": 61.54166793823242,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.4001933634281158,
+ "epoch": 7.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10428700596094131,
+ "kl": 0.0001954599174496252,
+ "learning_rate": 1.253106628854635e-07,
+ "loss": -0.08898806571960449,
+ "num_tokens": 8042035.0,
+ "reward": 0.21636998653411865,
+ "reward_std": 0.26617059111595154,
+ "rewards/true_env_reward_fn/mean": 0.21636998653411865,
+ "rewards/true_env_reward_fn/std": 0.26617059111595154,
+ "step": 326,
+ "step_time": 9.872497149000083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 69.91667175292969,
+ "completions/mean_terminated_length": 69.91667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.198440283536911,
+ "epoch": 7.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06512034684419632,
+ "kl": 9.893041169561911e-05,
+ "learning_rate": 1.2250560517925745e-07,
+ "loss": -0.004943478852510452,
+ "num_tokens": 8066991.0,
+ "reward": 0.5199041962623596,
+ "reward_std": 0.2916473150253296,
+ "rewards/true_env_reward_fn/mean": 0.5199041962623596,
+ "rewards/true_env_reward_fn/std": 0.2916473150253296,
+ "step": 327,
+ "step_time": 13.27990607999891
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.3655813038349152,
+ "epoch": 8.0,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08734066039323807,
+ "kl": 0.00011680843272188213,
+ "learning_rate": 1.197279099177731e-07,
+ "loss": -0.027742527425289154,
+ "num_tokens": 8095161.0,
+ "reward": 0.4403582811355591,
+ "reward_std": 0.20052418112754822,
+ "rewards/true_env_reward_fn/mean": 0.4403582811355591,
+ "rewards/true_env_reward_fn/std": 0.20052418112754822,
+ "step": 328,
+ "step_time": 9.091917235000437
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.645835876464844,
+ "completions/mean_terminated_length": 63.645835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1942758560180664,
+ "epoch": 8.024390243902438,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07103214412927628,
+ "kl": 9.241796033165883e-05,
+ "learning_rate": 1.1697777844051104e-07,
+ "loss": -0.024517521262168884,
+ "num_tokens": 8117752.0,
+ "reward": 0.4961467981338501,
+ "reward_std": 0.2681204676628113,
+ "rewards/true_env_reward_fn/mean": 0.4961467981338501,
+ "rewards/true_env_reward_fn/std": 0.2681204676628113,
+ "step": 329,
+ "step_time": 10.042522196999926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 71.58333587646484,
+ "completions/mean_terminated_length": 71.58333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.209133356809616,
+ "epoch": 8.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06019354239106178,
+ "kl": 2.6403215088066645e-05,
+ "learning_rate": 1.142554100890285e-07,
+ "loss": -0.03160044550895691,
+ "num_tokens": 8140164.0,
+ "reward": 0.5655641555786133,
+ "reward_std": 0.1564219743013382,
+ "rewards/true_env_reward_fn/mean": 0.5655641555786133,
+ "rewards/true_env_reward_fn/std": 0.1564219743013382,
+ "step": 330,
+ "step_time": 8.212663780001094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.229485958814621,
+ "epoch": 8.073170731707316,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07769559323787689,
+ "kl": 0.0001446352634957293,
+ "learning_rate": 1.115610021924902e-07,
+ "loss": -0.01835213601589203,
+ "num_tokens": 8177099.0,
+ "reward": 0.1968289315700531,
+ "reward_std": 0.3009001910686493,
+ "rewards/true_env_reward_fn/mean": 0.1968289166688919,
+ "rewards/true_env_reward_fn/std": 0.3009001612663269,
+ "step": 331,
+ "step_time": 16.508294159000798
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 55.72916793823242,
+ "completions/mean_terminated_length": 55.72916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.249097228050232,
+ "epoch": 8.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09750289469957352,
+ "kl": 0.00022061014351493213,
+ "learning_rate": 1.0889475005336446e-07,
+ "loss": 0.012823417782783508,
+ "num_tokens": 8202310.0,
+ "reward": 0.4960649907588959,
+ "reward_std": 0.28122010827064514,
+ "rewards/true_env_reward_fn/mean": 0.4960649907588959,
+ "rewards/true_env_reward_fn/std": 0.28122007846832275,
+ "step": 332,
+ "step_time": 9.309349606999604
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 326.0,
+ "completions/mean_length": 82.35417175292969,
+ "completions/mean_terminated_length": 73.21276092529297,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3090683817863464,
+ "epoch": 8.121951219512194,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09015674889087677,
+ "kl": 0.00010220101285085548,
+ "learning_rate": 1.0625684693326725e-07,
+ "loss": 0.23226313292980194,
+ "num_tokens": 8227663.0,
+ "reward": 0.4449918270111084,
+ "reward_std": 0.24675849080085754,
+ "rewards/true_env_reward_fn/mean": 0.4449918270111084,
+ "rewards/true_env_reward_fn/std": 0.24675849080085754,
+ "step": 333,
+ "step_time": 29.62071167800059
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 67.52083587646484,
+ "completions/mean_terminated_length": 67.52083587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2623873353004456,
+ "epoch": 8.146341463414634,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09357088804244995,
+ "kl": 0.00020532654161797836,
+ "learning_rate": 1.0364748403895368e-07,
+ "loss": 0.015600509941577911,
+ "num_tokens": 8251736.0,
+ "reward": 0.49391698837280273,
+ "reward_std": 0.313683420419693,
+ "rewards/true_env_reward_fn/mean": 0.49391698837280273,
+ "rewards/true_env_reward_fn/std": 0.3136834502220154,
+ "step": 334,
+ "step_time": 14.77577510500032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 89.25,
+ "completions/mean_terminated_length": 89.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.3438844978809357,
+ "epoch": 8.170731707317072,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07505634427070618,
+ "kl": 8.66996942932019e-05,
+ "learning_rate": 1.0106685050845837e-07,
+ "loss": -0.006955280434340239,
+ "num_tokens": 8283596.0,
+ "reward": 0.37073618173599243,
+ "reward_std": 0.22015252709388733,
+ "rewards/true_env_reward_fn/mean": 0.37073615193367004,
+ "rewards/true_env_reward_fn/std": 0.22015254199504852,
+ "step": 335,
+ "step_time": 14.388228956000603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3619366586208344,
+ "epoch": 8.195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07524604350328445,
+ "kl": 5.828110352013027e-05,
+ "learning_rate": 9.851513339738626e-08,
+ "loss": 0.02218097448348999,
+ "num_tokens": 8304798.0,
+ "reward": 0.6131325960159302,
+ "reward_std": 0.12095426768064499,
+ "rewards/true_env_reward_fn/mean": 0.6131325364112854,
+ "rewards/true_env_reward_fn/std": 0.12095426768064499,
+ "step": 336,
+ "step_time": 12.145640587000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 76.77083587646484,
+ "completions/mean_terminated_length": 67.51063537597656,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.6089566349983215,
+ "epoch": 8.21951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07641154527664185,
+ "kl": 0.00011577324767131358,
+ "learning_rate": 9.599251766535343e-08,
+ "loss": -0.047279007732868195,
+ "num_tokens": 8331715.0,
+ "reward": 0.37190374732017517,
+ "reward_std": 0.2730160653591156,
+ "rewards/true_env_reward_fn/mean": 0.37190374732017517,
+ "rewards/true_env_reward_fn/std": 0.2730160653591156,
+ "step": 337,
+ "step_time": 33.08387189299992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 60.79166793823242,
+ "completions/mean_terminated_length": 60.79166793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2664433717727661,
+ "epoch": 8.24390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10079872608184814,
+ "kl": 0.00016995913574646693,
+ "learning_rate": 9.349918616258113e-08,
+ "loss": 0.04466569796204567,
+ "num_tokens": 8357801.0,
+ "reward": 0.4597291648387909,
+ "reward_std": 0.2547009587287903,
+ "rewards/true_env_reward_fn/mean": 0.4597291648387909,
+ "rewards/true_env_reward_fn/std": 0.2547009289264679,
+ "step": 338,
+ "step_time": 12.540567380000539
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 278.0,
+ "completions/max_terminated_length": 278.0,
+ "completions/mean_length": 72.95833587646484,
+ "completions/mean_terminated_length": 72.95833587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.20101597905159,
+ "epoch": 8.268292682926829,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0724605992436409,
+ "kl": 8.693387826497201e-05,
+ "learning_rate": 9.103531961664118e-08,
+ "loss": -0.04201607406139374,
+ "num_tokens": 8391047.0,
+ "reward": 0.272682249546051,
+ "reward_std": 0.22370731830596924,
+ "rewards/true_env_reward_fn/mean": 0.27268221974372864,
+ "rewards/true_env_reward_fn/std": 0.22370733320713043,
+ "step": 339,
+ "step_time": 21.499952529999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 66.27083587646484,
+ "completions/mean_terminated_length": 66.27083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.240132600069046,
+ "epoch": 8.292682926829269,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09206120669841766,
+ "kl": 9.196988685289398e-05,
+ "learning_rate": 8.860109661935672e-08,
+ "loss": 0.013995083048939705,
+ "num_tokens": 8412060.0,
+ "reward": 0.5260962843894958,
+ "reward_std": 0.16053001582622528,
+ "rewards/true_env_reward_fn/mean": 0.5260962843894958,
+ "rewards/true_env_reward_fn/std": 0.16053001582622528,
+ "step": 340,
+ "step_time": 11.287615471999743
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 187.0,
+ "completions/max_terminated_length": 187.0,
+ "completions/mean_length": 69.85417175292969,
+ "completions/mean_terminated_length": 69.85417175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2434665858745575,
+ "epoch": 8.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06606078892946243,
+ "kl": 0.00013524454698199406,
+ "learning_rate": 8.619669361385662e-08,
+ "loss": 0.04330434650182724,
+ "num_tokens": 8435189.0,
+ "reward": 0.5729736089706421,
+ "reward_std": 0.28772154450416565,
+ "rewards/true_env_reward_fn/mean": 0.5729735493659973,
+ "rewards/true_env_reward_fn/std": 0.28772154450416565,
+ "step": 341,
+ "step_time": 14.554103153000142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 87.83333587646484,
+ "completions/mean_terminated_length": 87.83333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.193087100982666,
+ "epoch": 8.341463414634147,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06404980272054672,
+ "kl": 7.958761125337332e-05,
+ "learning_rate": 8.382228488178638e-08,
+ "loss": -0.02654789574444294,
+ "num_tokens": 8461485.0,
+ "reward": 0.4391756057739258,
+ "reward_std": 0.3396260142326355,
+ "rewards/true_env_reward_fn/mean": 0.4391756057739258,
+ "rewards/true_env_reward_fn/std": 0.3396260142326355,
+ "step": 342,
+ "step_time": 16.93046250899988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2198111712932587,
+ "epoch": 8.365853658536585,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06700363755226135,
+ "kl": 3.631071058407542e-05,
+ "learning_rate": 8.14780425306758e-08,
+ "loss": -0.01248769462108612,
+ "num_tokens": 8477128.0,
+ "reward": 0.6880009174346924,
+ "reward_std": 0.16646623611450195,
+ "rewards/true_env_reward_fn/mean": 0.6880009174346924,
+ "rewards/true_env_reward_fn/std": 0.16646623611450195,
+ "step": 343,
+ "step_time": 8.379146702999606
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 72.0625,
+ "completions/mean_terminated_length": 72.0625,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.365002065896988,
+ "epoch": 8.390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07387524098157883,
+ "kl": 7.906068458396476e-05,
+ "learning_rate": 7.91641364814628e-08,
+ "loss": -0.019427858293056488,
+ "num_tokens": 8502659.0,
+ "reward": 0.49987101554870605,
+ "reward_std": 0.22534418106079102,
+ "rewards/true_env_reward_fn/mean": 0.49987101554870605,
+ "rewards/true_env_reward_fn/std": 0.22534416615962982,
+ "step": 344,
+ "step_time": 16.447989433999282
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 63.833335876464844,
+ "completions/mean_terminated_length": 63.833335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3033632636070251,
+ "epoch": 8.414634146341463,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07896492630243301,
+ "kl": 0.00015640226229152177,
+ "learning_rate": 7.688073445617798e-08,
+ "loss": 0.032075606286525726,
+ "num_tokens": 8527771.0,
+ "reward": 0.4142627418041229,
+ "reward_std": 0.39965084195137024,
+ "rewards/true_env_reward_fn/mean": 0.4142627418041229,
+ "rewards/true_env_reward_fn/std": 0.3996508717536926,
+ "step": 345,
+ "step_time": 10.625701701998878
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 222.0,
+ "completions/max_terminated_length": 222.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2410016655921936,
+ "epoch": 8.439024390243903,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10202007740736008,
+ "kl": 0.00028172876773169264,
+ "learning_rate": 7.462800196578661e-08,
+ "loss": 0.030375666916370392,
+ "num_tokens": 8557085.0,
+ "reward": 0.3401162028312683,
+ "reward_std": 0.2810371518135071,
+ "rewards/true_env_reward_fn/mean": 0.3401161730289459,
+ "rewards/true_env_reward_fn/std": 0.2810371518135071,
+ "step": 346,
+ "step_time": 15.773468264999792
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 62.97916793823242,
+ "completions/mean_terminated_length": 62.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4290962517261505,
+ "epoch": 8.463414634146341,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06221809610724449,
+ "kl": 9.756034887686837e-05,
+ "learning_rate": 7.240610229819195e-08,
+ "loss": -0.009350121021270752,
+ "num_tokens": 8583644.0,
+ "reward": 0.42270374298095703,
+ "reward_std": 0.19876612722873688,
+ "rewards/true_env_reward_fn/mean": 0.42270374298095703,
+ "rewards/true_env_reward_fn/std": 0.19876612722873688,
+ "step": 347,
+ "step_time": 10.18261290999908
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 361.0,
+ "completions/max_terminated_length": 361.0,
+ "completions/mean_length": 68.89583587646484,
+ "completions/mean_terminated_length": 68.89583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2232879400253296,
+ "epoch": 8.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07711587101221085,
+ "kl": 7.56127965360065e-05,
+ "learning_rate": 7.021519650639951e-08,
+ "loss": -0.06937319040298462,
+ "num_tokens": 8604055.0,
+ "reward": 0.5483574867248535,
+ "reward_std": 0.2410479635000229,
+ "rewards/true_env_reward_fn/mean": 0.5483574867248535,
+ "rewards/true_env_reward_fn/std": 0.2410479635000229,
+ "step": 348,
+ "step_time": 20.953229555999314
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.2409018576145172,
+ "epoch": 8.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08634211868047714,
+ "kl": 0.000148440574776032,
+ "learning_rate": 6.805544339684293e-08,
+ "loss": 0.05703897029161453,
+ "num_tokens": 8636849.0,
+ "reward": 0.2619777023792267,
+ "reward_std": 0.3443836271762848,
+ "rewards/true_env_reward_fn/mean": 0.2619777023792267,
+ "rewards/true_env_reward_fn/std": 0.3443836271762848,
+ "step": 349,
+ "step_time": 13.73512468699937
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 64.85417175292969,
+ "completions/mean_terminated_length": 64.85417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2963348627090454,
+ "epoch": 8.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09569360315799713,
+ "kl": 0.0001818603413994424,
+ "learning_rate": 6.592699951787362e-08,
+ "loss": 0.0750311091542244,
+ "num_tokens": 8668938.0,
+ "reward": 0.2865946292877197,
+ "reward_std": 0.21787859499454498,
+ "rewards/true_env_reward_fn/mean": 0.2865946292877197,
+ "rewards/true_env_reward_fn/std": 0.21787859499454498,
+ "step": 350,
+ "step_time": 16.08931997000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.66666793823242,
+ "completions/mean_terminated_length": 62.66666793823242,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2852083146572113,
+ "epoch": 8.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08515980839729309,
+ "kl": 0.00022268274369707797,
+ "learning_rate": 6.383001914841252e-08,
+ "loss": -0.003520023077726364,
+ "num_tokens": 8686442.0,
+ "reward": 0.5473253726959229,
+ "reward_std": 0.2312658727169037,
+ "rewards/true_env_reward_fn/mean": 0.5473253726959229,
+ "rewards/true_env_reward_fn/std": 0.2312658578157425,
+ "step": 351,
+ "step_time": 9.42530540899952
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 58.60416793823242,
+ "completions/mean_terminated_length": 58.60416793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1837059557437897,
+ "epoch": 8.585365853658537,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372796326875687,
+ "kl": 7.27423093849211e-05,
+ "learning_rate": 6.176465428676819e-08,
+ "loss": 0.003352940082550049,
+ "num_tokens": 8705967.0,
+ "reward": 0.5402672290802002,
+ "reward_std": 0.2537318766117096,
+ "rewards/true_env_reward_fn/mean": 0.5402672290802002,
+ "rewards/true_env_reward_fn/std": 0.253731906414032,
+ "step": 352,
+ "step_time": 13.651303137998184
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3596981465816498,
+ "epoch": 8.609756097560975,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09085690230131149,
+ "kl": 0.00015744064694445115,
+ "learning_rate": 5.973105463961864e-08,
+ "loss": -0.03212148696184158,
+ "num_tokens": 8733134.0,
+ "reward": 0.4029428958892822,
+ "reward_std": 0.2143087089061737,
+ "rewards/true_env_reward_fn/mean": 0.4029428958892822,
+ "rewards/true_env_reward_fn/std": 0.2143087238073349,
+ "step": 353,
+ "step_time": 12.830564498999593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.225245863199234,
+ "epoch": 8.634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11574450135231018,
+ "kl": 0.00027416441298555583,
+ "learning_rate": 5.772936761116026e-08,
+ "loss": 0.05132449418306351,
+ "num_tokens": 8761296.0,
+ "reward": 0.35531648993492126,
+ "reward_std": 0.20710644125938416,
+ "rewards/true_env_reward_fn/mean": 0.35531648993492126,
+ "rewards/true_env_reward_fn/std": 0.20710645616054535,
+ "step": 354,
+ "step_time": 11.43733696199888
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 59.72916793823242,
+ "completions/mean_terminated_length": 59.72916793823242,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1999734044075012,
+ "epoch": 8.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08676803112030029,
+ "kl": 0.0001471151099394774,
+ "learning_rate": 5.575973829242364e-08,
+ "loss": -0.007696744054555893,
+ "num_tokens": 8786723.0,
+ "reward": 0.3953551948070526,
+ "reward_std": 0.3271177411079407,
+ "rewards/true_env_reward_fn/mean": 0.3953551948070526,
+ "rewards/true_env_reward_fn/std": 0.32711780071258545,
+ "step": 355,
+ "step_time": 10.52069242699963
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 64.14583587646484,
+ "completions/mean_terminated_length": 64.14583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2520077526569366,
+ "epoch": 8.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08498971164226532,
+ "kl": 0.00012545567187771667,
+ "learning_rate": 5.382230945075556e-08,
+ "loss": 0.034280166029930115,
+ "num_tokens": 8809690.0,
+ "reward": 0.42336568236351013,
+ "reward_std": 0.2602991759777069,
+ "rewards/true_env_reward_fn/mean": 0.42336568236351013,
+ "rewards/true_env_reward_fn/std": 0.2602991759777069,
+ "step": 356,
+ "step_time": 11.943708853999851
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 67.45833587646484,
+ "completions/mean_terminated_length": 67.45833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1441007107496262,
+ "epoch": 8.707317073170731,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05060715973377228,
+ "kl": 9.935822527040727e-05,
+ "learning_rate": 5.191722151947225e-08,
+ "loss": 0.002798471599817276,
+ "num_tokens": 8836960.0,
+ "reward": 0.39054369926452637,
+ "reward_std": 0.20666222274303436,
+ "rewards/true_env_reward_fn/mean": 0.39054369926452637,
+ "rewards/true_env_reward_fn/std": 0.20666222274303436,
+ "step": 357,
+ "step_time": 9.9503519280006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 74.625,
+ "completions/mean_terminated_length": 65.31914520263672,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2771258652210236,
+ "epoch": 8.731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07117093354463577,
+ "kl": 0.00011358128540450707,
+ "learning_rate": 5.004461258767872e-08,
+ "loss": -0.1029033362865448,
+ "num_tokens": 8867398.0,
+ "reward": 0.46532535552978516,
+ "reward_std": 0.16995222866535187,
+ "rewards/true_env_reward_fn/mean": 0.46532535552978516,
+ "rewards/true_env_reward_fn/std": 0.16995222866535187,
+ "step": 358,
+ "step_time": 36.7129373509988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3237419724464417,
+ "epoch": 8.75609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08377347886562347,
+ "kl": 0.00010999429287039675,
+ "learning_rate": 4.820461839026046e-08,
+ "loss": 0.05175921320915222,
+ "num_tokens": 8889378.0,
+ "reward": 0.5536255240440369,
+ "reward_std": 0.26559942960739136,
+ "rewards/true_env_reward_fn/mean": 0.5536255240440369,
+ "rewards/true_env_reward_fn/std": 0.26559942960739136,
+ "step": 359,
+ "step_time": 11.28046775799885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 74.10417175292969,
+ "completions/mean_terminated_length": 74.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3808394372463226,
+ "epoch": 8.78048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08288726955652237,
+ "kl": 0.0001290821719521773,
+ "learning_rate": 4.639737229804402e-08,
+ "loss": -0.007075890898704529,
+ "num_tokens": 8910959.0,
+ "reward": 0.5237338542938232,
+ "reward_std": 0.27443575859069824,
+ "rewards/true_env_reward_fn/mean": 0.5237338542938232,
+ "rewards/true_env_reward_fn/std": 0.27443575859069824,
+ "step": 360,
+ "step_time": 13.746602035998876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2947768867015839,
+ "epoch": 8.804878048780488,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048931583762168884,
+ "kl": 7.009085675235838e-05,
+ "learning_rate": 4.462300530813024e-08,
+ "loss": 0.041338130831718445,
+ "num_tokens": 8929517.0,
+ "reward": 0.568915069103241,
+ "reward_std": 0.17454475164413452,
+ "rewards/true_env_reward_fn/mean": 0.568915069103241,
+ "rewards/true_env_reward_fn/std": 0.17454475164413452,
+ "step": 361,
+ "step_time": 8.730673061999369
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 328.0,
+ "completions/max_terminated_length": 328.0,
+ "completions/mean_length": 67.95833587646484,
+ "completions/mean_terminated_length": 67.95833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2548468708992004,
+ "epoch": 8.829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07735498249530792,
+ "kl": 4.255911517248023e-05,
+ "learning_rate": 4.288164603439892e-08,
+ "loss": -0.0545659139752388,
+ "num_tokens": 8950043.0,
+ "reward": 0.5133440494537354,
+ "reward_std": 0.29375824332237244,
+ "rewards/true_env_reward_fn/mean": 0.5133439898490906,
+ "rewards/true_env_reward_fn/std": 0.29375824332237244,
+ "step": 362,
+ "step_time": 22.50767488800011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2430709302425385,
+ "epoch": 8.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08263012021780014,
+ "kl": 0.00014031877253728453,
+ "learning_rate": 4.117342069818602e-08,
+ "loss": 0.045923154801130295,
+ "num_tokens": 8972927.0,
+ "reward": 0.5072638988494873,
+ "reward_std": 0.20991377532482147,
+ "rewards/true_env_reward_fn/mean": 0.5072638988494873,
+ "rewards/true_env_reward_fn/std": 0.20991379022598267,
+ "step": 363,
+ "step_time": 14.428374270999484
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 55.29166793823242,
+ "completions/mean_terminated_length": 55.29166793823242,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.120811641216278,
+ "epoch": 8.878048780487806,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08290416747331619,
+ "kl": 0.00016724429042369593,
+ "learning_rate": 3.949845311913491e-08,
+ "loss": -0.00451173260807991,
+ "num_tokens": 8988781.0,
+ "reward": 0.6316336393356323,
+ "reward_std": 0.22976641356945038,
+ "rewards/true_env_reward_fn/mean": 0.6316335797309875,
+ "rewards/true_env_reward_fn/std": 0.22976639866828918,
+ "step": 364,
+ "step_time": 7.905863810999108
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1943224370479584,
+ "epoch": 8.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07621228694915771,
+ "kl": 6.38052747490292e-05,
+ "learning_rate": 3.785686470622118e-08,
+ "loss": -0.0022124722599983215,
+ "num_tokens": 9007073.0,
+ "reward": 0.5937620401382446,
+ "reward_std": 0.1718287616968155,
+ "rewards/true_env_reward_fn/mean": 0.5937620401382446,
+ "rewards/true_env_reward_fn/std": 0.1718287467956543,
+ "step": 365,
+ "step_time": 9.652277058999061
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 57.5625,
+ "completions/mean_terminated_length": 57.5625,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2130553424358368,
+ "epoch": 8.926829268292684,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07400088012218475,
+ "kl": 0.00011410296792746522,
+ "learning_rate": 3.624877444895269e-08,
+ "loss": 0.01366148516535759,
+ "num_tokens": 9032908.0,
+ "reward": 0.4561588168144226,
+ "reward_std": 0.31542423367500305,
+ "rewards/true_env_reward_fn/mean": 0.4561587870121002,
+ "rewards/true_env_reward_fn/std": 0.31542423367500305,
+ "step": 366,
+ "step_time": 10.32316389099924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 77.25,
+ "completions/mean_terminated_length": 77.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3448135256767273,
+ "epoch": 8.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07419808954000473,
+ "kl": 9.254619635612471e-05,
+ "learning_rate": 3.467429890874424e-08,
+ "loss": 0.017107762396335602,
+ "num_tokens": 9058768.0,
+ "reward": 0.5096973180770874,
+ "reward_std": 0.15345345437526703,
+ "rewards/true_env_reward_fn/mean": 0.5096972584724426,
+ "rewards/true_env_reward_fn/std": 0.15345345437526703,
+ "step": 367,
+ "step_time": 14.400456866999775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 251.0,
+ "completions/max_terminated_length": 251.0,
+ "completions/mean_length": 76.75,
+ "completions/mean_terminated_length": 76.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.329552710056305,
+ "epoch": 8.975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.057363200932741165,
+ "kl": 9.755073915584944e-05,
+ "learning_rate": 3.3133552210468874e-08,
+ "loss": 0.008944882079958916,
+ "num_tokens": 9085804.0,
+ "reward": 0.3852631747722626,
+ "reward_std": 0.22340475022792816,
+ "rewards/true_env_reward_fn/mean": 0.3852631747722626,
+ "rewards/true_env_reward_fn/std": 0.22340476512908936,
+ "step": 368,
+ "step_time": 17.320541074000175
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 65.66667175292969,
+ "completions/mean_terminated_length": 65.66667175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2402052581310272,
+ "epoch": 9.0,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08145138621330261,
+ "kl": 0.00013890985428588465,
+ "learning_rate": 3.162664603418608e-08,
+ "loss": 0.0042404308915138245,
+ "num_tokens": 9109756.0,
+ "reward": 0.5699147582054138,
+ "reward_std": 0.2426392138004303,
+ "rewards/true_env_reward_fn/mean": 0.5699147582054138,
+ "rewards/true_env_reward_fn/std": 0.2426392287015915,
+ "step": 369,
+ "step_time": 14.103951319000771
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.234388917684555,
+ "epoch": 9.024390243902438,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08930502086877823,
+ "kl": 0.00010845370161405299,
+ "learning_rate": 3.015368960704584e-08,
+ "loss": -0.007921114563941956,
+ "num_tokens": 9128316.0,
+ "reward": 0.6464049220085144,
+ "reward_std": 0.2783248722553253,
+ "rewards/true_env_reward_fn/mean": 0.6464049220085144,
+ "rewards/true_env_reward_fn/std": 0.2783248722553253,
+ "step": 370,
+ "step_time": 13.493940920000568
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3062849640846252,
+ "epoch": 9.048780487804878,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05373607575893402,
+ "kl": 9.319536547991447e-05,
+ "learning_rate": 2.8714789695372054e-08,
+ "loss": 0.003740280866622925,
+ "num_tokens": 9149964.0,
+ "reward": 0.5325896739959717,
+ "reward_std": 0.2599954307079315,
+ "rewards/true_env_reward_fn/mean": 0.5325896739959717,
+ "rewards/true_env_reward_fn/std": 0.2599954307079315,
+ "step": 371,
+ "step_time": 9.884774114998436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 68.14583587646484,
+ "completions/mean_terminated_length": 68.14583587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2394476234912872,
+ "epoch": 9.073170731707316,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07394114136695862,
+ "kl": 0.0001138780153269181,
+ "learning_rate": 2.731005059692332e-08,
+ "loss": -0.0023280810564756393,
+ "num_tokens": 9170259.0,
+ "reward": 0.5012303590774536,
+ "reward_std": 0.2768687903881073,
+ "rewards/true_env_reward_fn/mean": 0.5012302994728088,
+ "rewards/true_env_reward_fn/std": 0.2768687605857849,
+ "step": 372,
+ "step_time": 9.664578052998877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 65.25,
+ "completions/mean_terminated_length": 65.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2490745186805725,
+ "epoch": 9.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06408056616783142,
+ "kl": 5.751862272518338e-05,
+ "learning_rate": 2.5939574133333308e-08,
+ "loss": -0.013112705200910568,
+ "num_tokens": 9197423.0,
+ "reward": 0.42906099557876587,
+ "reward_std": 0.29850679636001587,
+ "rewards/true_env_reward_fn/mean": 0.4290609657764435,
+ "rewards/true_env_reward_fn/std": 0.29850679636001587,
+ "step": 373,
+ "step_time": 10.526233300999593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 83.47917175292969,
+ "completions/mean_terminated_length": 83.47917175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.333191066980362,
+ "epoch": 9.121951219512194,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0637340396642685,
+ "kl": 0.00014988034035923192,
+ "learning_rate": 2.4603459642729864e-08,
+ "loss": 0.008196331560611725,
+ "num_tokens": 9224102.0,
+ "reward": 0.4708556830883026,
+ "reward_std": 0.2077488750219345,
+ "rewards/true_env_reward_fn/mean": 0.4708556830883026,
+ "rewards/true_env_reward_fn/std": 0.2077488750219345,
+ "step": 374,
+ "step_time": 13.534195267000541
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 272.0,
+ "completions/max_terminated_length": 272.0,
+ "completions/mean_length": 67.77083587646484,
+ "completions/mean_terminated_length": 67.77083587646484,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.151135116815567,
+ "epoch": 9.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07668020576238632,
+ "kl": 3.856326316054037e-05,
+ "learning_rate": 2.3301803972534728e-08,
+ "loss": 0.05261360481381416,
+ "num_tokens": 9252339.0,
+ "reward": 0.379050612449646,
+ "reward_std": 0.23492643237113953,
+ "rewards/true_env_reward_fn/mean": 0.379050612449646,
+ "rewards/true_env_reward_fn/std": 0.23492641746997833,
+ "step": 375,
+ "step_time": 24.720070157999544
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.322300672531128,
+ "epoch": 9.170731707317072,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11152477562427521,
+ "kl": 0.00033667498792055994,
+ "learning_rate": 2.2034701472443852e-08,
+ "loss": -0.038920704275369644,
+ "num_tokens": 9270227.0,
+ "reward": 0.5742875933647156,
+ "reward_std": 0.2614876925945282,
+ "rewards/true_env_reward_fn/mean": 0.5742875933647156,
+ "rewards/true_env_reward_fn/std": 0.2614877223968506,
+ "step": 376,
+ "step_time": 9.753906144999746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 67.3125,
+ "completions/mean_terminated_length": 67.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.322103351354599,
+ "epoch": 9.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06792955845594406,
+ "kl": 0.00011458320295787416,
+ "learning_rate": 2.0802243987588064e-08,
+ "loss": 0.08347739279270172,
+ "num_tokens": 9299602.0,
+ "reward": 0.4416275918483734,
+ "reward_std": 0.31829217076301575,
+ "rewards/true_env_reward_fn/mean": 0.4416275918483734,
+ "rewards/true_env_reward_fn/std": 0.31829220056533813,
+ "step": 377,
+ "step_time": 13.209823182000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 63.6875,
+ "completions/mean_terminated_length": 63.6875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3833276331424713,
+ "epoch": 9.21951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08396119624376297,
+ "kl": 0.00019161509044351988,
+ "learning_rate": 1.9604520851876194e-08,
+ "loss": 0.01698162779211998,
+ "num_tokens": 9331507.0,
+ "reward": 0.26609930396080017,
+ "reward_std": 0.25199094414711,
+ "rewards/true_env_reward_fn/mean": 0.26609930396080017,
+ "rewards/true_env_reward_fn/std": 0.2519909739494324,
+ "step": 378,
+ "step_time": 15.5939898680017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 62.645835876464844,
+ "completions/mean_terminated_length": 62.645835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.287186712026596,
+ "epoch": 9.24390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0844208225607872,
+ "kl": 6.250779006222729e-05,
+ "learning_rate": 1.8441618881519184e-08,
+ "loss": -0.0004980191588401794,
+ "num_tokens": 9352346.0,
+ "reward": 0.6220773458480835,
+ "reward_std": 0.17538678646087646,
+ "rewards/true_env_reward_fn/mean": 0.6220772862434387,
+ "rewards/true_env_reward_fn/std": 0.17538677155971527,
+ "step": 379,
+ "step_time": 11.331410343000243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3174393773078918,
+ "epoch": 9.268292682926829,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08529601246118546,
+ "kl": 0.0001577844341227319,
+ "learning_rate": 1.7313622368738013e-08,
+ "loss": 0.02950325235724449,
+ "num_tokens": 9381101.0,
+ "reward": 0.3922676742076874,
+ "reward_std": 0.2629895806312561,
+ "rewards/true_env_reward_fn/mean": 0.3922676742076874,
+ "rewards/true_env_reward_fn/std": 0.2629896104335785,
+ "step": 380,
+ "step_time": 11.785110086000714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 55.833335876464844,
+ "completions/mean_terminated_length": 55.833335876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2294963598251343,
+ "epoch": 9.292682926829269,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08615893125534058,
+ "kl": 0.00012760807658196427,
+ "learning_rate": 1.62206130756532e-08,
+ "loss": 0.03380218520760536,
+ "num_tokens": 9402133.0,
+ "reward": 0.48932167887687683,
+ "reward_std": 0.3009134531021118,
+ "rewards/true_env_reward_fn/mean": 0.48932167887687683,
+ "rewards/true_env_reward_fn/std": 0.3009134531021118,
+ "step": 381,
+ "step_time": 11.71825248299865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 70.3125,
+ "completions/mean_terminated_length": 70.3125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2500975131988525,
+ "epoch": 9.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08025979995727539,
+ "kl": 0.00015432167128892615,
+ "learning_rate": 1.51626702283586e-08,
+ "loss": -0.026728281751275063,
+ "num_tokens": 9433948.0,
+ "reward": 0.3480832576751709,
+ "reward_std": 0.27214208245277405,
+ "rewards/true_env_reward_fn/mean": 0.3480832576751709,
+ "rewards/true_env_reward_fn/std": 0.27214211225509644,
+ "step": 382,
+ "step_time": 11.077961950000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 207.0,
+ "completions/max_terminated_length": 207.0,
+ "completions/mean_length": 77.83333587646484,
+ "completions/mean_terminated_length": 77.83333587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2940633893013,
+ "epoch": 9.341463414634147,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07732700556516647,
+ "kl": 0.00013818908701068722,
+ "learning_rate": 1.4139870511178765e-08,
+ "loss": 0.012543938122689724,
+ "num_tokens": 9457780.0,
+ "reward": 0.4787862300872803,
+ "reward_std": 0.2881976068019867,
+ "rewards/true_env_reward_fn/mean": 0.4787862300872803,
+ "rewards/true_env_reward_fn/std": 0.2881976068019867,
+ "step": 383,
+ "step_time": 14.209161775999746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 65.64583587646484,
+ "completions/mean_terminated_length": 65.64583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.25718092918396,
+ "epoch": 9.365853658536585,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10170208662748337,
+ "kl": 0.00013350013250601478,
+ "learning_rate": 1.3152288061110517e-08,
+ "loss": -0.013931870460510254,
+ "num_tokens": 9475411.0,
+ "reward": 0.6825136542320251,
+ "reward_std": 0.21768879890441895,
+ "rewards/true_env_reward_fn/mean": 0.6825136542320251,
+ "rewards/true_env_reward_fn/std": 0.21768878400325775,
+ "step": 384,
+ "step_time": 8.900575762000699
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 206.0,
+ "completions/max_terminated_length": 206.0,
+ "completions/mean_length": 67.83333587646484,
+ "completions/mean_terminated_length": 67.83333587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2512764632701874,
+ "epoch": 9.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06906364113092422,
+ "kl": 5.4121877838042565e-05,
+ "learning_rate": 1.2199994462448904e-08,
+ "loss": -0.01152697205543518,
+ "num_tokens": 9492995.0,
+ "reward": 0.5913000106811523,
+ "reward_std": 0.24868597090244293,
+ "rewards/true_env_reward_fn/mean": 0.5913000106811523,
+ "rewards/true_env_reward_fn/std": 0.24868597090244293,
+ "step": 385,
+ "step_time": 13.630191889000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.348086029291153,
+ "epoch": 9.414634146341463,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06427548080682755,
+ "kl": 6.0526374909386504e-05,
+ "learning_rate": 1.128305874159896e-08,
+ "loss": -0.039815668016672134,
+ "num_tokens": 9516749.0,
+ "reward": 0.48740649223327637,
+ "reward_std": 0.29916781187057495,
+ "rewards/true_env_reward_fn/mean": 0.48740649223327637,
+ "rewards/true_env_reward_fn/std": 0.29916781187057495,
+ "step": 386,
+ "step_time": 17.153756488999534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.64583587646484,
+ "completions/mean_terminated_length": 75.64583587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3448026180267334,
+ "epoch": 9.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.04732977971434593,
+ "kl": 5.01183408232464e-05,
+ "learning_rate": 1.0401547362071938e-08,
+ "loss": -0.03205295279622078,
+ "num_tokens": 9545300.0,
+ "reward": 0.43718665838241577,
+ "reward_std": 0.16248689591884613,
+ "rewards/true_env_reward_fn/mean": 0.4371866285800934,
+ "rewards/true_env_reward_fn/std": 0.16248688101768494,
+ "step": 387,
+ "step_time": 11.892045039999175
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 57.458335876464844,
+ "completions/mean_terminated_length": 57.458335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1685180962085724,
+ "epoch": 9.463414634146341,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0918080136179924,
+ "kl": 0.00025482301134616137,
+ "learning_rate": 9.555524219667988e-09,
+ "loss": 0.0153394415974617,
+ "num_tokens": 9568906.0,
+ "reward": 0.46309399604797363,
+ "reward_std": 0.27685311436653137,
+ "rewards/true_env_reward_fn/mean": 0.46309399604797363,
+ "rewards/true_env_reward_fn/std": 0.276853084564209,
+ "step": 388,
+ "step_time": 9.755774155000836
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 167.0,
+ "completions/max_terminated_length": 167.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.268288493156433,
+ "epoch": 9.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09713966399431229,
+ "kl": 0.00021595886028080713,
+ "learning_rate": 8.745050637844532e-09,
+ "loss": 0.10685908794403076,
+ "num_tokens": 9596667.0,
+ "reward": 0.375690221786499,
+ "reward_std": 0.27006518840789795,
+ "rewards/true_env_reward_fn/mean": 0.375690221786499,
+ "rewards/true_env_reward_fn/std": 0.27006518840789795,
+ "step": 389,
+ "step_time": 13.222017186999437
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 69.20833587646484,
+ "completions/mean_terminated_length": 69.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2847715020179749,
+ "epoch": 9.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08115239441394806,
+ "kl": 0.0001902375151985325,
+ "learning_rate": 7.970185363271431e-09,
+ "loss": -0.015931159257888794,
+ "num_tokens": 9620285.0,
+ "reward": 0.5215861797332764,
+ "reward_std": 0.29267051815986633,
+ "rewards/true_env_reward_fn/mean": 0.5215861797332764,
+ "rewards/true_env_reward_fn/std": 0.29267051815986633,
+ "step": 390,
+ "step_time": 19.701098540999737
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 75.70833587646484,
+ "completions/mean_terminated_length": 75.70833587646484,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.1211588382720947,
+ "epoch": 9.536585365853659,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.050371747463941574,
+ "kl": 5.3559015668724896e-05,
+ "learning_rate": 7.230984561572729e-09,
+ "loss": -0.027369298040866852,
+ "num_tokens": 9643879.0,
+ "reward": 0.5257008075714111,
+ "reward_std": 0.31278616189956665,
+ "rewards/true_env_reward_fn/mean": 0.5257008075714111,
+ "rewards/true_env_reward_fn/std": 0.31278616189956665,
+ "step": 391,
+ "step_time": 14.5430295999995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.22917175292969,
+ "completions/mean_terminated_length": 73.22917175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.162130355834961,
+ "epoch": 9.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06951762735843658,
+ "kl": 0.00011329071639920585,
+ "learning_rate": 6.5275018132553425e-09,
+ "loss": 0.005165033042430878,
+ "num_tokens": 9674170.0,
+ "reward": 0.38153380155563354,
+ "reward_std": 0.2553096115589142,
+ "rewards/true_env_reward_fn/mean": 0.38153377175331116,
+ "rewards/true_env_reward_fn/std": 0.2553096115589142,
+ "step": 392,
+ "step_time": 13.884568534998834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1723724007606506,
+ "epoch": 9.585365853658537,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06391289085149765,
+ "kl": 0.00012617898028111085,
+ "learning_rate": 5.859788109825792e-09,
+ "loss": -0.026335250586271286,
+ "num_tokens": 9697471.0,
+ "reward": 0.5592976808547974,
+ "reward_std": 0.294358491897583,
+ "rewards/true_env_reward_fn/mean": 0.5592976212501526,
+ "rewards/true_env_reward_fn/std": 0.294358491897583,
+ "step": 393,
+ "step_time": 10.88813568399928
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 75.0625,
+ "completions/mean_terminated_length": 75.0625,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2699165344238281,
+ "epoch": 9.609756097560975,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0821971669793129,
+ "kl": 0.00011502238521643449,
+ "learning_rate": 5.2278918500933134e-09,
+ "loss": 0.020743347704410553,
+ "num_tokens": 9727482.0,
+ "reward": 0.3525441288948059,
+ "reward_std": 0.24436385929584503,
+ "rewards/true_env_reward_fn/mean": 0.3525440990924835,
+ "rewards/true_env_reward_fn/std": 0.24436385929584503,
+ "step": 394,
+ "step_time": 13.137662936999732
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 64.14583587646484,
+ "completions/mean_terminated_length": 64.14583587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2177735269069672,
+ "epoch": 9.634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08585982769727707,
+ "kl": 9.731890622788342e-05,
+ "learning_rate": 4.631858836662561e-09,
+ "loss": -0.0022293110378086567,
+ "num_tokens": 9752601.0,
+ "reward": 0.42186734080314636,
+ "reward_std": 0.14065590500831604,
+ "rewards/true_env_reward_fn/mean": 0.42186734080314636,
+ "rewards/true_env_reward_fn/std": 0.14065590500831604,
+ "step": 395,
+ "step_time": 9.551430144000733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 72.95833587646484,
+ "completions/mean_terminated_length": 72.95833587646484,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3700671792030334,
+ "epoch": 9.658536585365853,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0687054842710495,
+ "kl": 0.00012583125862875022,
+ "learning_rate": 4.071732272613148e-09,
+ "loss": -0.010379405692219734,
+ "num_tokens": 9776159.0,
+ "reward": 0.40437865257263184,
+ "reward_std": 0.20715568959712982,
+ "rewards/true_env_reward_fn/mean": 0.40437865257263184,
+ "rewards/true_env_reward_fn/std": 0.20715567469596863,
+ "step": 396,
+ "step_time": 9.576685387999532
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 65.0625,
+ "completions/mean_terminated_length": 65.0625,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.1619998216629028,
+ "epoch": 9.682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06019222363829613,
+ "kl": 8.781285851000575e-05,
+ "learning_rate": 3.5475527583680997e-09,
+ "loss": -0.023738304153084755,
+ "num_tokens": 9797522.0,
+ "reward": 0.5591601729393005,
+ "reward_std": 0.2965308129787445,
+ "rewards/true_env_reward_fn/mean": 0.5591601729393005,
+ "rewards/true_env_reward_fn/std": 0.2965308427810669,
+ "step": 397,
+ "step_time": 12.962693579000188
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 66.39583587646484,
+ "completions/mean_terminated_length": 56.91489028930664,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.0172521024942398,
+ "epoch": 9.707317073170731,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.02889176644384861,
+ "kl": 7.828448997315718e-05,
+ "learning_rate": 3.0593582887512014e-09,
+ "loss": 0.11688737571239471,
+ "num_tokens": 9817717.0,
+ "reward": 0.5341037511825562,
+ "reward_std": 0.33641698956489563,
+ "rewards/true_env_reward_fn/mean": 0.5341036915779114,
+ "rewards/true_env_reward_fn/std": 0.33641698956489563,
+ "step": 398,
+ "step_time": 30.514281355000094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 383.0,
+ "completions/max_terminated_length": 383.0,
+ "completions/mean_length": 70.72917175292969,
+ "completions/mean_terminated_length": 70.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2152343690395355,
+ "epoch": 9.731707317073171,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07666711509227753,
+ "kl": 0.00014014396219863556,
+ "learning_rate": 2.6071842502326526e-09,
+ "loss": -0.051372602581977844,
+ "num_tokens": 9847608.0,
+ "reward": 0.46223634481430054,
+ "reward_std": 0.21379408240318298,
+ "rewards/true_env_reward_fn/mean": 0.46223631501197815,
+ "rewards/true_env_reward_fn/std": 0.21379409730434418,
+ "step": 399,
+ "step_time": 24.57003485899986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.272740215063095,
+ "epoch": 9.75609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06314582377672195,
+ "kl": 0.0001023452778099454,
+ "learning_rate": 2.1910634183644472e-09,
+ "loss": 0.0021672993898391724,
+ "num_tokens": 9869730.0,
+ "reward": 0.41130340099334717,
+ "reward_std": 0.35641229152679443,
+ "rewards/true_env_reward_fn/mean": 0.41130340099334717,
+ "rewards/true_env_reward_fn/std": 0.3564123213291168,
+ "step": 400,
+ "step_time": 12.93115923699952
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 410,
+ "num_input_tokens_seen": 9869730,
+ "num_train_epochs": 10,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 12,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-400/training_args.bin b/checkpoint-400/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..2276397b5f715b1fed46fa8d458d3bde360f7b81
--- /dev/null
+++ b/checkpoint-400/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
+size 6776
diff --git a/checkpoint-410/README.md b/checkpoint-410/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-410/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-410/adapter_config.json b/checkpoint-410/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-410/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-410/adapter_model.safetensors b/checkpoint-410/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0bee0aed05d4191a3acdedb0dc86170d1654847c
--- /dev/null
+++ b/checkpoint-410/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebdd83e68b5fdd48ae1bba30bc5d3100fe03446c96aea23fa9b9f3e1083e446a
+size 8731128
diff --git a/checkpoint-410/chat_template.jinja b/checkpoint-410/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-410/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-410/optimizer.pt b/checkpoint-410/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be64bcff1aca308a39046644fc982c03a2d3b0a7
--- /dev/null
+++ b/checkpoint-410/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bae134aaf77dd28afcaec77f29b6f501666f4652dfcf7a76cec80582be3e408d
+size 17526842
diff --git a/checkpoint-410/ref/adapter_config.json b/checkpoint-410/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-410/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-410/ref/adapter_model.safetensors b/checkpoint-410/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4b516b95e2bde01b4a51b7977bce639f00946144
--- /dev/null
+++ b/checkpoint-410/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
+size 8731128
diff --git a/checkpoint-410/rng_state.pth b/checkpoint-410/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..84688fb18a567f5dfce5267b20f7894728d39d3f
--- /dev/null
+++ b/checkpoint-410/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92f7258275803ee225c48d9f0be24a2e804322d8d71dffb01017bd88acea4aa9
+size 14244
diff --git a/checkpoint-410/scaler.pt b/checkpoint-410/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e7220a9cf734d1d5f8f9d7b95a6dbd51095fab1
--- /dev/null
+++ b/checkpoint-410/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b493e733a8107a2f848b886ae6979bbc174fff60d33308c53f3641abf2499428
+size 988
diff --git a/checkpoint-410/scheduler.pt b/checkpoint-410/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5baf17ae353291a09a3fe48532a741c530821f87
--- /dev/null
+++ b/checkpoint-410/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdb873b724e5a65ec168778540eb0c67065ce086c5fa83d417852ef12a4e9cbd
+size 1064
diff --git a/checkpoint-410/tokenizer.json b/checkpoint-410/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-410/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-410/tokenizer_config.json b/checkpoint-410/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-410/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-410/trainer_state.json b/checkpoint-410/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c96c34cb2fab457dbf8b7f87b91da802c5111027
--- /dev/null
+++ b/checkpoint-410/trainer_state.json
@@ -0,0 +1,11514 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 10.0,
+ "eval_steps": 500,
+ "global_step": 410,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
+ "learning_rate": 0.0,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
+ "step": 1,
+ "step_time": 11.815711200999885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
+ "step": 2,
+ "step_time": 13.493524850000085
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
+ "step": 3,
+ "step_time": 11.449303891999875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
+ "step": 4,
+ "step_time": 18.835909622000145
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
+ "step": 5,
+ "step_time": 14.435845696000001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
+ "step": 6,
+ "step_time": 13.232063896999989
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
+ "step": 7,
+ "step_time": 9.810652986000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
+ "step": 8,
+ "step_time": 19.874756868999953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
+ "step": 9,
+ "step_time": 9.458149736999985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
+ "step": 10,
+ "step_time": 14.219840567000006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
+ "step": 11,
+ "step_time": 9.271131832999913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
+ "step": 12,
+ "step_time": 12.184364300000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
+ "step": 13,
+ "step_time": 10.720424850000086
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
+ "step": 14,
+ "step_time": 14.965493325000011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
+ "step": 15,
+ "step_time": 13.628413805999912
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
+ "step": 16,
+ "step_time": 14.229579036999894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
+ "step": 17,
+ "step_time": 17.023353198999985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
+ "step": 18,
+ "step_time": 11.965533113999868
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
+ "step": 19,
+ "step_time": 10.33812468799988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
+ "step": 20,
+ "step_time": 17.317542748000164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
+ "step": 21,
+ "step_time": 12.20283881399996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
+ "step": 22,
+ "step_time": 11.211206898000114
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
+ "step": 23,
+ "step_time": 10.428452587999914
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
+ "step": 24,
+ "step_time": 11.223491792000118
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
+ "step": 25,
+ "step_time": 10.92509102200006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
+ "step": 26,
+ "step_time": 10.458724108999945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
+ "step": 27,
+ "step_time": 14.364785926000081
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
+ "step": 28,
+ "step_time": 10.888908384999922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
+ "step": 29,
+ "step_time": 15.510035302999995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
+ "step": 30,
+ "step_time": 14.50245602599989
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
+ "step": 31,
+ "step_time": 11.06186091799998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
+ "step": 32,
+ "step_time": 15.808748693000211
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
+ "step": 33,
+ "step_time": 24.20358999299981
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
+ "step": 34,
+ "step_time": 12.218407348999904
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
+ "step": 35,
+ "step_time": 9.66908789599995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
+ "step": 36,
+ "step_time": 8.370980583999994
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
+ "step": 37,
+ "step_time": 10.285125336999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
+ "step": 38,
+ "step_time": 12.449738128000035
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
+ "step": 39,
+ "step_time": 10.279209028999958
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
+ "step": 40,
+ "step_time": 13.58010066599968
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
+ "step": 41,
+ "step_time": 10.35077203700007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
+ "step": 42,
+ "step_time": 10.548370664999993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
+ "step": 43,
+ "step_time": 24.089396637999698
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
+ "step": 44,
+ "step_time": 19.39071501599983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
+ "step": 45,
+ "step_time": 13.893569495999827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
+ "step": 46,
+ "step_time": 10.50698806499986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
+ "step": 47,
+ "step_time": 12.964419044000124
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
+ "step": 48,
+ "step_time": 17.23041258299986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
+ "step": 49,
+ "step_time": 8.691208415999881
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
+ "step": 50,
+ "step_time": 10.162109979000206
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2760189771652222,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0870513767004013,
+ "kl": 1.4371181578098913e-05,
+ "learning_rate": 9.985329005918702e-07,
+ "loss": -0.01623840071260929,
+ "num_tokens": 1253120.0,
+ "reward": 0.3888077139854431,
+ "reward_std": 0.3346175253391266,
+ "rewards/true_env_reward_fn/mean": 0.3888076841831207,
+ "rewards/true_env_reward_fn/std": 0.3346175253391266,
+ "step": 51,
+ "step_time": 10.88732858200001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 52.72916793823242,
+ "completions/mean_terminated_length": 52.72916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1669773161411285,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.1055479422211647,
+ "kl": 1.69047059443983e-05,
+ "learning_rate": 9.981889740445957e-07,
+ "loss": 0.03519687056541443,
+ "num_tokens": 1274803.0,
+ "reward": 0.4995749592781067,
+ "reward_std": 0.2088174670934677,
+ "rewards/true_env_reward_fn/mean": 0.4995749294757843,
+ "rewards/true_env_reward_fn/std": 0.2088174819946289,
+ "step": 52,
+ "step_time": 9.252596976000177
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 310.0,
+ "completions/max_terminated_length": 310.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2451297044754028,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07513763755559921,
+ "kl": 1.5911174841676257e-05,
+ "learning_rate": 9.978089365816355e-07,
+ "loss": 0.07006432861089706,
+ "num_tokens": 1305910.0,
+ "reward": 0.33895593881607056,
+ "reward_std": 0.2969740927219391,
+ "rewards/true_env_reward_fn/mean": 0.33895590901374817,
+ "rewards/true_env_reward_fn/std": 0.2969740927219391,
+ "step": 53,
+ "step_time": 24.22518693999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3110129833221436,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0695035383105278,
+ "kl": 1.606306568646687e-05,
+ "learning_rate": 9.973928157497674e-07,
+ "loss": 0.03299739956855774,
+ "num_tokens": 1330815.0,
+ "reward": 0.4440445899963379,
+ "reward_std": 0.2889502942562103,
+ "rewards/true_env_reward_fn/mean": 0.4440445899963379,
+ "rewards/true_env_reward_fn/std": 0.2889502942562103,
+ "step": 54,
+ "step_time": 10.14821418500037
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 61.958335876464844,
+ "completions/mean_terminated_length": 61.958335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2969173192977905,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09475436061620712,
+ "kl": 1.5850717545617954e-05,
+ "learning_rate": 9.969406417112488e-07,
+ "loss": -0.014009319245815277,
+ "num_tokens": 1361885.0,
+ "reward": 0.3289160430431366,
+ "reward_std": 0.26591774821281433,
+ "rewards/true_env_reward_fn/mean": 0.3289160430431366,
+ "rewards/true_env_reward_fn/std": 0.2659177780151367,
+ "step": 55,
+ "step_time": 11.13082981000025
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 80.22917175292969,
+ "completions/mean_terminated_length": 80.22917175292969,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2630544006824493,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.06734384596347809,
+ "kl": 1.5091616887730197e-05,
+ "learning_rate": 9.964524472416317e-07,
+ "loss": -0.10958556830883026,
+ "num_tokens": 1390496.0,
+ "reward": 0.46485185623168945,
+ "reward_std": 0.29441413283348083,
+ "rewards/true_env_reward_fn/mean": 0.46485185623168945,
+ "rewards/true_env_reward_fn/std": 0.29441413283348083,
+ "step": 56,
+ "step_time": 14.49393488900023
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 64.27083587646484,
+ "completions/mean_terminated_length": 64.27083587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1640427708625793,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07240130007266998,
+ "kl": 1.509602225269191e-05,
+ "learning_rate": 9.959282677273868e-07,
+ "loss": 0.10520926117897034,
+ "num_tokens": 1411837.0,
+ "reward": 0.5296112895011902,
+ "reward_std": 0.2505757212638855,
+ "rewards/true_env_reward_fn/mean": 0.5296112895011902,
+ "rewards/true_env_reward_fn/std": 0.2505757212638855,
+ "step": 57,
+ "step_time": 14.065935286000013
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 65.58333587646484,
+ "completions/mean_terminated_length": 65.58333587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1222519278526306,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05992415174841881,
+ "kl": 1.2099166724510724e-05,
+ "learning_rate": 9.953681411633374e-07,
+ "loss": 0.004622246604412794,
+ "num_tokens": 1438569.0,
+ "reward": 0.41778087615966797,
+ "reward_std": 0.28395572304725647,
+ "rewards/true_env_reward_fn/mean": 0.41778087615966797,
+ "rewards/true_env_reward_fn/std": 0.2839556932449341,
+ "step": 58,
+ "step_time": 12.76291504100027
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 75.33333587646484,
+ "completions/mean_terminated_length": 75.33333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1821540892124176,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0654020607471466,
+ "kl": 1.4932766589481616e-05,
+ "learning_rate": 9.947721081499067e-07,
+ "loss": 0.06719422340393066,
+ "num_tokens": 1461033.0,
+ "reward": 0.5268458127975464,
+ "reward_std": 0.23783695697784424,
+ "rewards/true_env_reward_fn/mean": 0.5268457531929016,
+ "rewards/true_env_reward_fn/std": 0.23783694207668304,
+ "step": 59,
+ "step_time": 11.089177285999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 72.14583587646484,
+ "completions/mean_terminated_length": 72.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2446446418762207,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08000538498163223,
+ "kl": 1.3416995898296591e-05,
+ "learning_rate": 9.941402118901742e-07,
+ "loss": 0.05287330970168114,
+ "num_tokens": 1488264.0,
+ "reward": 0.4032561779022217,
+ "reward_std": 0.24067741632461548,
+ "rewards/true_env_reward_fn/mean": 0.4032561779022217,
+ "rewards/true_env_reward_fn/std": 0.24067740142345428,
+ "step": 60,
+ "step_time": 12.328215124000053
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 191.0,
+ "completions/max_terminated_length": 191.0,
+ "completions/mean_length": 65.8125,
+ "completions/mean_terminated_length": 65.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.416578859090805,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07916785031557083,
+ "kl": 1.8312134670850355e-05,
+ "learning_rate": 9.934724981867446e-07,
+ "loss": -0.02956264466047287,
+ "num_tokens": 1506607.0,
+ "reward": 0.6846215724945068,
+ "reward_std": 0.21603551506996155,
+ "rewards/true_env_reward_fn/mean": 0.6846215724945068,
+ "rewards/true_env_reward_fn/std": 0.21603552997112274,
+ "step": 61,
+ "step_time": 13.294195681999781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 182.0,
+ "completions/max_terminated_length": 182.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2478635609149933,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09035050123929977,
+ "kl": 1.867344440142915e-05,
+ "learning_rate": 9.927690154384272e-07,
+ "loss": -0.048415351659059525,
+ "num_tokens": 1539351.0,
+ "reward": 0.17628252506256104,
+ "reward_std": 0.2993278205394745,
+ "rewards/true_env_reward_fn/mean": 0.17628252506256104,
+ "rewards/true_env_reward_fn/std": 0.2993278503417969,
+ "step": 62,
+ "step_time": 18.972790307999958
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 61.66666793823242,
+ "completions/mean_terminated_length": 61.66666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2024545669555664,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09254598617553711,
+ "kl": 1.654068455536617e-05,
+ "learning_rate": 9.920298146367286e-07,
+ "loss": 0.09414710104465485,
+ "num_tokens": 1565215.0,
+ "reward": 0.4147046208381653,
+ "reward_std": 0.1770697683095932,
+ "rewards/true_env_reward_fn/mean": 0.4147045910358429,
+ "rewards/true_env_reward_fn/std": 0.1770697683095932,
+ "step": 63,
+ "step_time": 11.292631820999986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 64.1875,
+ "completions/mean_terminated_length": 64.1875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2442612051963806,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.052295491099357605,
+ "kl": 1.716045289867907e-05,
+ "learning_rate": 9.912549493621554e-07,
+ "loss": 0.01475335843861103,
+ "num_tokens": 1589608.0,
+ "reward": 0.48741206526756287,
+ "reward_std": 0.21404753625392914,
+ "rewards/true_env_reward_fn/mean": 0.48741206526756287,
+ "rewards/true_env_reward_fn/std": 0.21404753625392914,
+ "step": 64,
+ "step_time": 9.600786530999585
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 61.3125,
+ "completions/mean_terminated_length": 61.3125,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2735203504562378,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.094636932015419,
+ "kl": 2.017962560785236e-05,
+ "learning_rate": 9.90444475780332e-07,
+ "loss": -0.0016674790531396866,
+ "num_tokens": 1619095.0,
+ "reward": 0.34675830602645874,
+ "reward_std": 0.2556215822696686,
+ "rewards/true_env_reward_fn/mean": 0.34675827622413635,
+ "rewards/true_env_reward_fn/std": 0.2556215822696686,
+ "step": 65,
+ "step_time": 10.169144185999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 65.9375,
+ "completions/mean_terminated_length": 65.9375,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2845994234085083,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05164559930562973,
+ "kl": 1.66792838172114e-05,
+ "learning_rate": 9.89598452637928e-07,
+ "loss": 0.011961851269006729,
+ "num_tokens": 1645076.0,
+ "reward": 0.44275379180908203,
+ "reward_std": 0.2063576877117157,
+ "rewards/true_env_reward_fn/mean": 0.44275379180908203,
+ "rewards/true_env_reward_fn/std": 0.2063576877117157,
+ "step": 66,
+ "step_time": 10.343706631000032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3418844938278198,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05729615315794945,
+ "kl": 1.6437259546364658e-05,
+ "learning_rate": 9.88716941258401e-07,
+ "loss": 0.015346314758062363,
+ "num_tokens": 1675423.0,
+ "reward": 0.4190921187400818,
+ "reward_std": 0.3388116955757141,
+ "rewards/true_env_reward_fn/mean": 0.4190920889377594,
+ "rewards/true_env_reward_fn/std": 0.3388116657733917,
+ "step": 67,
+ "step_time": 12.16719347000003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 78.66667175292969,
+ "completions/mean_terminated_length": 78.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2716343402862549,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07959166914224625,
+ "kl": 1.785568679224525e-05,
+ "learning_rate": 9.87800005537551e-07,
+ "loss": 0.08119910955429077,
+ "num_tokens": 1698103.0,
+ "reward": 0.5178458094596863,
+ "reward_std": 0.22635267674922943,
+ "rewards/true_env_reward_fn/mean": 0.5178458094596863,
+ "rewards/true_env_reward_fn/std": 0.22635267674922943,
+ "step": 68,
+ "step_time": 15.523659553000016
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.1755708158016205,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08657841384410858,
+ "kl": 2.0373249526528525e-05,
+ "learning_rate": 9.868477119388894e-07,
+ "loss": -0.01668858528137207,
+ "num_tokens": 1723155.0,
+ "reward": 0.39579567313194275,
+ "reward_std": 0.3625684082508087,
+ "rewards/true_env_reward_fn/mean": 0.39579567313194275,
+ "rewards/true_env_reward_fn/std": 0.3625684380531311,
+ "step": 69,
+ "step_time": 9.677535032999685
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 74.79167175292969,
+ "completions/mean_terminated_length": 74.79167175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.296934336423874,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07736476510763168,
+ "kl": 2.0918997506669257e-05,
+ "learning_rate": 9.85860129488821e-07,
+ "loss": 0.03239107131958008,
+ "num_tokens": 1742689.0,
+ "reward": 0.6141302585601807,
+ "reward_std": 0.23138943314552307,
+ "rewards/true_env_reward_fn/mean": 0.6141302585601807,
+ "rewards/true_env_reward_fn/std": 0.23138941824436188,
+ "step": 70,
+ "step_time": 15.20990351499995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2145576775074005,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07104668766260147,
+ "kl": 1.726100731502811e-05,
+ "learning_rate": 9.848373297716414e-07,
+ "loss": 0.03256790712475777,
+ "num_tokens": 1765463.0,
+ "reward": 0.48419874906539917,
+ "reward_std": 0.32040080428123474,
+ "rewards/true_env_reward_fn/mean": 0.4841987192630768,
+ "rewards/true_env_reward_fn/std": 0.32040080428123474,
+ "step": 71,
+ "step_time": 14.703903473999844
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 62.35416793823242,
+ "completions/mean_terminated_length": 62.35416793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.265857070684433,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07747533172369003,
+ "kl": 1.5618132920280914e-05,
+ "learning_rate": 9.837793869243467e-07,
+ "loss": -0.00018438976258039474,
+ "num_tokens": 1791512.0,
+ "reward": 0.45079630613327026,
+ "reward_std": 0.2226068526506424,
+ "rewards/true_env_reward_fn/mean": 0.4507962763309479,
+ "rewards/true_env_reward_fn/std": 0.2226068526506424,
+ "step": 72,
+ "step_time": 9.422353613000269
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 73.0625,
+ "completions/mean_terminated_length": 73.0625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3557232320308685,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0811411589384079,
+ "kl": 1.9600361611082917e-05,
+ "learning_rate": 9.826863776312618e-07,
+ "loss": -0.019779374822974205,
+ "num_tokens": 1820731.0,
+ "reward": 0.431186318397522,
+ "reward_std": 0.23306044936180115,
+ "rewards/true_env_reward_fn/mean": 0.431186318397522,
+ "rewards/true_env_reward_fn/std": 0.23306044936180115,
+ "step": 73,
+ "step_time": 16.65922043799992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 78.375,
+ "completions/mean_terminated_length": 78.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2767037451267242,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08133924007415771,
+ "kl": 1.8058163732348476e-05,
+ "learning_rate": 9.815583811184808e-07,
+ "loss": -0.02447839081287384,
+ "num_tokens": 1841389.0,
+ "reward": 0.5825158953666687,
+ "reward_std": 0.2041907161474228,
+ "rewards/true_env_reward_fn/mean": 0.5825158953666687,
+ "rewards/true_env_reward_fn/std": 0.20419073104858398,
+ "step": 74,
+ "step_time": 13.422026366999944
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2325710952281952,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11014537513256073,
+ "kl": 3.268667387601454e-05,
+ "learning_rate": 9.803954791481238e-07,
+ "loss": 0.045359574258327484,
+ "num_tokens": 1871129.0,
+ "reward": 0.3935621678829193,
+ "reward_std": 0.22456605732440948,
+ "rewards/true_env_reward_fn/mean": 0.3935621678829193,
+ "rewards/true_env_reward_fn/std": 0.2245660424232483,
+ "step": 75,
+ "step_time": 9.51117546200021
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 74.85417175292969,
+ "completions/mean_terminated_length": 74.85417175292969,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.412343978881836,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07616850733757019,
+ "kl": 1.848336046350596e-05,
+ "learning_rate": 9.791977560124118e-07,
+ "loss": 0.030123719945549965,
+ "num_tokens": 1892706.0,
+ "reward": 0.5764689445495605,
+ "reward_std": 0.18864154815673828,
+ "rewards/true_env_reward_fn/mean": 0.5764689445495605,
+ "rewards/true_env_reward_fn/std": 0.18864154815673828,
+ "step": 76,
+ "step_time": 9.295928349999713
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 71.54167175292969,
+ "completions/mean_terminated_length": 71.54167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.342492938041687,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08141017705202103,
+ "kl": 1.587149881743244e-05,
+ "learning_rate": 9.779652985275562e-07,
+ "loss": -0.02852344512939453,
+ "num_tokens": 1915324.0,
+ "reward": 0.4926157593727112,
+ "reward_std": 0.20701222121715546,
+ "rewards/true_env_reward_fn/mean": 0.4926157295703888,
+ "rewards/true_env_reward_fn/std": 0.20701222121715546,
+ "step": 77,
+ "step_time": 15.693113021000045
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 62.3125,
+ "completions/mean_terminated_length": 62.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2102001011371613,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0555732287466526,
+ "kl": 1.6820984001242323e-05,
+ "learning_rate": 9.766981960274652e-07,
+ "loss": 0.041817761957645416,
+ "num_tokens": 1933867.0,
+ "reward": 0.5576165318489075,
+ "reward_std": 0.3197881579399109,
+ "rewards/true_env_reward_fn/mean": 0.5576165318489075,
+ "rewards/true_env_reward_fn/std": 0.3197881281375885,
+ "step": 78,
+ "step_time": 16.146651725000083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 69.25,
+ "completions/mean_terminated_length": 69.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2909597754478455,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05769016966223717,
+ "kl": 1.534885905130068e-05,
+ "learning_rate": 9.753965403572702e-07,
+ "loss": -0.04179058223962784,
+ "num_tokens": 1953375.0,
+ "reward": 0.5617212057113647,
+ "reward_std": 0.18222570419311523,
+ "rewards/true_env_reward_fn/mean": 0.5617212057113647,
+ "rewards/true_env_reward_fn/std": 0.18222568929195404,
+ "step": 79,
+ "step_time": 9.82867347299998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.2647078335285187,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06353812664747238,
+ "kl": 2.195177648900426e-05,
+ "learning_rate": 9.740604258666668e-07,
+ "loss": -0.09541463106870651,
+ "num_tokens": 1978255.0,
+ "reward": 0.5184200406074524,
+ "reward_std": 0.28920501470565796,
+ "rewards/true_env_reward_fn/mean": 0.5184200406074524,
+ "rewards/true_env_reward_fn/std": 0.28920501470565796,
+ "step": 80,
+ "step_time": 11.267316974000096
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 62.72916793823242,
+ "completions/mean_terminated_length": 62.72916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3497782051563263,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08975031226873398,
+ "kl": 3.0107988550298614e-05,
+ "learning_rate": 9.726899494030766e-07,
+ "loss": 0.04644065350294113,
+ "num_tokens": 2007634.0,
+ "reward": 0.3841831684112549,
+ "reward_std": 0.30559155344963074,
+ "rewards/true_env_reward_fn/mean": 0.3841831684112549,
+ "rewards/true_env_reward_fn/std": 0.30559155344963074,
+ "step": 81,
+ "step_time": 10.035370067999793
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 64.91667175292969,
+ "completions/mean_terminated_length": 64.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2758312821388245,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08747493475675583,
+ "kl": 1.806905720513896e-05,
+ "learning_rate": 9.71285210304628e-07,
+ "loss": -0.07698298245668411,
+ "num_tokens": 2024382.0,
+ "reward": 0.6160596609115601,
+ "reward_std": 0.23944181203842163,
+ "rewards/true_env_reward_fn/mean": 0.6160596609115601,
+ "rewards/true_env_reward_fn/std": 0.23944182693958282,
+ "step": 82,
+ "step_time": 9.56242024800008
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 63.0625,
+ "completions/mean_terminated_length": 63.0625,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2893573343753815,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05648891627788544,
+ "kl": 1.960936606337782e-05,
+ "learning_rate": 9.698463103929541e-07,
+ "loss": 0.05065512657165527,
+ "num_tokens": 2046817.0,
+ "reward": 0.5863184332847595,
+ "reward_std": 0.19063502550125122,
+ "rewards/true_env_reward_fn/mean": 0.5863184332847595,
+ "rewards/true_env_reward_fn/std": 0.19063502550125122,
+ "step": 83,
+ "step_time": 10.563381390999666
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 75.29167175292969,
+ "completions/mean_terminated_length": 75.29167175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2169642746448517,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06511837989091873,
+ "kl": 2.2800771603215253e-05,
+ "learning_rate": 9.683733539658138e-07,
+ "loss": 0.02157626487314701,
+ "num_tokens": 2074535.0,
+ "reward": 0.4389227330684662,
+ "reward_std": 0.303769588470459,
+ "rewards/true_env_reward_fn/mean": 0.4389227330684662,
+ "rewards/true_env_reward_fn/std": 0.303769588470459,
+ "step": 84,
+ "step_time": 17.21621736799966
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 432.0,
+ "completions/max_terminated_length": 432.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.396474927663803,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06777605414390564,
+ "kl": 2.369298363191774e-05,
+ "learning_rate": 9.66866447789531e-07,
+ "loss": -0.024554546922445297,
+ "num_tokens": 2096031.0,
+ "reward": 0.5134819746017456,
+ "reward_std": 0.28690314292907715,
+ "rewards/true_env_reward_fn/mean": 0.5134819149971008,
+ "rewards/true_env_reward_fn/std": 0.28690314292907715,
+ "step": 85,
+ "step_time": 25.519813745999954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.95833587646484,
+ "completions/mean_terminated_length": 65.95833587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3048341274261475,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07558907568454742,
+ "kl": 1.8465078937879298e-05,
+ "learning_rate": 9.653257010912558e-07,
+ "loss": -0.015101172029972076,
+ "num_tokens": 2122829.0,
+ "reward": 0.4031229019165039,
+ "reward_std": 0.22406692802906036,
+ "rewards/true_env_reward_fn/mean": 0.4031229019165039,
+ "rewards/true_env_reward_fn/std": 0.22406692802906036,
+ "step": 86,
+ "step_time": 10.78625990699993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 82.125,
+ "completions/mean_terminated_length": 82.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3657839596271515,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07850468903779984,
+ "kl": 2.0332241774667636e-05,
+ "learning_rate": 9.637512255510474e-07,
+ "loss": 0.06651890277862549,
+ "num_tokens": 2151091.0,
+ "reward": 0.3940638303756714,
+ "reward_std": 0.2639860212802887,
+ "rewards/true_env_reward_fn/mean": 0.3940638303756714,
+ "rewards/true_env_reward_fn/std": 0.2639860212802887,
+ "step": 87,
+ "step_time": 13.604215705999877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 58.680850982666016,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.0381308495998383,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06663572043180466,
+ "kl": 2.4382573428738397e-05,
+ "learning_rate": 9.621431352937787e-07,
+ "loss": -0.08434788882732391,
+ "num_tokens": 2177281.0,
+ "reward": 0.40229034423828125,
+ "reward_std": 0.3266920745372772,
+ "rewards/true_env_reward_fn/mean": 0.40229034423828125,
+ "rewards/true_env_reward_fn/std": 0.3266920745372772,
+ "step": 88,
+ "step_time": 32.408574500999975
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0944073796272278,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06779129058122635,
+ "kl": 2.3317856630455935e-05,
+ "learning_rate": 9.60501546880865e-07,
+ "loss": 0.019480882212519646,
+ "num_tokens": 2200208.0,
+ "reward": 0.5087729692459106,
+ "reward_std": 0.33071935176849365,
+ "rewards/true_env_reward_fn/mean": 0.5087729096412659,
+ "rewards/true_env_reward_fn/std": 0.33071935176849365,
+ "step": 89,
+ "step_time": 9.901715897000031
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1683936715126038,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07440414279699326,
+ "kl": 1.8814549775925116e-05,
+ "learning_rate": 9.58826579301814e-07,
+ "loss": -0.03402159363031387,
+ "num_tokens": 2227789.0,
+ "reward": 0.40219685435295105,
+ "reward_std": 0.17851270735263824,
+ "rewards/true_env_reward_fn/mean": 0.40219685435295105,
+ "rewards/true_env_reward_fn/std": 0.17851269245147705,
+ "step": 90,
+ "step_time": 11.152492722000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 68.22917175292969,
+ "completions/mean_terminated_length": 68.22917175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1628780961036682,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03311198577284813,
+ "kl": 1.5094836498974473e-05,
+ "learning_rate": 9.57118353965601e-07,
+ "loss": 0.01087917946279049,
+ "num_tokens": 2252192.0,
+ "reward": 0.5357927083969116,
+ "reward_std": 0.18703003227710724,
+ "rewards/true_env_reward_fn/mean": 0.5357926487922668,
+ "rewards/true_env_reward_fn/std": 0.18703003227710724,
+ "step": 91,
+ "step_time": 10.656350811000038
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 66.97917175292969,
+ "completions/mean_terminated_length": 66.97917175292969,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2369268834590912,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08958107978105545,
+ "kl": 2.8437810669856844e-05,
+ "learning_rate": 9.553769946918698e-07,
+ "loss": 0.005673397332429886,
+ "num_tokens": 2274199.0,
+ "reward": 0.5484694242477417,
+ "reward_std": 0.27515000104904175,
+ "rewards/true_env_reward_fn/mean": 0.5484693646430969,
+ "rewards/true_env_reward_fn/std": 0.27515000104904175,
+ "step": 92,
+ "step_time": 10.304143018000104
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 69.0625,
+ "completions/mean_terminated_length": 69.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.374023586511612,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03936443477869034,
+ "kl": 1.8785845441016136e-05,
+ "learning_rate": 9.53602627701956e-07,
+ "loss": -0.01821933500468731,
+ "num_tokens": 2302818.0,
+ "reward": 0.3851678967475891,
+ "reward_std": 0.2433396279811859,
+ "rewards/true_env_reward_fn/mean": 0.3851678669452667,
+ "rewards/true_env_reward_fn/std": 0.2433396428823471,
+ "step": 93,
+ "step_time": 13.589426085000014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 68.9375,
+ "completions/mean_terminated_length": 68.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3004788756370544,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06094004213809967,
+ "kl": 1.9176507976226276e-05,
+ "learning_rate": 9.517953816097395e-07,
+ "loss": 0.023817429319024086,
+ "num_tokens": 2325071.0,
+ "reward": 0.6004310846328735,
+ "reward_std": 0.23291133344173431,
+ "rewards/true_env_reward_fn/mean": 0.6004310250282288,
+ "rewards/true_env_reward_fn/std": 0.23291133344173431,
+ "step": 94,
+ "step_time": 14.587356482000132
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 237.0,
+ "completions/max_terminated_length": 237.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.233375996351242,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08650019764900208,
+ "kl": 1.89352349480032e-05,
+ "learning_rate": 9.499553874123212e-07,
+ "loss": 0.14434456825256348,
+ "num_tokens": 2347902.0,
+ "reward": 0.5542359948158264,
+ "reward_std": 0.18165862560272217,
+ "rewards/true_env_reward_fn/mean": 0.5542359948158264,
+ "rewards/true_env_reward_fn/std": 0.18165862560272217,
+ "step": 95,
+ "step_time": 14.689755582000089
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 57.0625,
+ "completions/mean_terminated_length": 57.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.224440723657608,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07974361628293991,
+ "kl": 1.838593607317307e-05,
+ "learning_rate": 9.480827784805278e-07,
+ "loss": 0.03995979577302933,
+ "num_tokens": 2361401.0,
+ "reward": 0.6956334114074707,
+ "reward_std": 0.185209721326828,
+ "rewards/true_env_reward_fn/mean": 0.6956334114074707,
+ "rewards/true_env_reward_fn/std": 0.185209721326828,
+ "step": 96,
+ "step_time": 10.379233056999965
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.330334097146988,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08950946480035782,
+ "kl": 3.245086418246501e-05,
+ "learning_rate": 9.461776905492444e-07,
+ "loss": -0.03975849226117134,
+ "num_tokens": 2384437.0,
+ "reward": 0.49323582649230957,
+ "reward_std": 0.30376356840133667,
+ "rewards/true_env_reward_fn/mean": 0.49323582649230957,
+ "rewards/true_env_reward_fn/std": 0.3037635385990143,
+ "step": 97,
+ "step_time": 10.037491584999998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 63.8125,
+ "completions/mean_terminated_length": 63.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2434260249137878,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09637262672185898,
+ "kl": 3.597719251047238e-05,
+ "learning_rate": 9.442402617075764e-07,
+ "loss": 0.008840052410960197,
+ "num_tokens": 2409676.0,
+ "reward": 0.47345292568206787,
+ "reward_std": 0.3432519733905792,
+ "rewards/true_env_reward_fn/mean": 0.47345292568206787,
+ "rewards/true_env_reward_fn/std": 0.34325194358825684,
+ "step": 98,
+ "step_time": 13.073343929999965
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3253428936004639,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1034398227930069,
+ "kl": 3.889948129653931e-05,
+ "learning_rate": 9.422706323888396e-07,
+ "loss": 0.01636725291609764,
+ "num_tokens": 2433369.0,
+ "reward": 0.5016611218452454,
+ "reward_std": 0.3056275546550751,
+ "rewards/true_env_reward_fn/mean": 0.5016611218452454,
+ "rewards/true_env_reward_fn/std": 0.3056274950504303,
+ "step": 99,
+ "step_time": 9.465850557000067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 62.125,
+ "completions/mean_terminated_length": 62.125,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.2484558820724487,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08669883012771606,
+ "kl": 1.584698543410923e-05,
+ "learning_rate": 9.402689453603814e-07,
+ "loss": 0.13139240443706512,
+ "num_tokens": 2458407.0,
+ "reward": 0.34693777561187744,
+ "reward_std": 0.35830602049827576,
+ "rewards/true_env_reward_fn/mean": 0.34693777561187744,
+ "rewards/true_env_reward_fn/std": 0.35830605030059814,
+ "step": 100,
+ "step_time": 11.33050741000011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 68.77083587646484,
+ "completions/mean_terminated_length": 68.77083587646484,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.2351897060871124,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06004978343844414,
+ "kl": 2.0037293097630027e-05,
+ "learning_rate": 9.382353457132317e-07,
+ "loss": -0.04131116345524788,
+ "num_tokens": 2483052.0,
+ "reward": 0.38015443086624146,
+ "reward_std": 0.34710174798965454,
+ "rewards/true_env_reward_fn/mean": 0.38015440106391907,
+ "rewards/true_env_reward_fn/std": 0.34710174798965454,
+ "step": 101,
+ "step_time": 16.478299477000064
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3605049848556519,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09010742604732513,
+ "kl": 3.810847783825011e-05,
+ "learning_rate": 9.361699808515875e-07,
+ "loss": 0.038716960698366165,
+ "num_tokens": 2510193.0,
+ "reward": 0.3458574712276459,
+ "reward_std": 0.30283215641975403,
+ "rewards/true_env_reward_fn/mean": 0.3458574712276459,
+ "rewards/true_env_reward_fn/std": 0.30283215641975403,
+ "step": 102,
+ "step_time": 11.344593008000174
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.19815993309021,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07395386695861816,
+ "kl": 2.6301003344997298e-05,
+ "learning_rate": 9.340730004821265e-07,
+ "loss": 0.01458972692489624,
+ "num_tokens": 2529212.0,
+ "reward": 0.5586616396903992,
+ "reward_std": 0.20150764286518097,
+ "rewards/true_env_reward_fn/mean": 0.5586616396903992,
+ "rewards/true_env_reward_fn/std": 0.20150764286518097,
+ "step": 103,
+ "step_time": 8.135681302999728
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.89583587646484,
+ "completions/mean_terminated_length": 73.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2152214348316193,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08900879323482513,
+ "kl": 3.066915814997628e-05,
+ "learning_rate": 9.31944556603157e-07,
+ "loss": 0.08802390843629837,
+ "num_tokens": 2557007.0,
+ "reward": 0.4009184241294861,
+ "reward_std": 0.32733896374702454,
+ "rewards/true_env_reward_fn/mean": 0.4009183943271637,
+ "rewards/true_env_reward_fn/std": 0.3273389935493469,
+ "step": 104,
+ "step_time": 15.185034105999875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.39583587646484,
+ "completions/mean_terminated_length": 68.39583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2849501073360443,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0703769251704216,
+ "kl": 1.9505746195136453e-05,
+ "learning_rate": 9.297848034936005e-07,
+ "loss": 0.036192238330841064,
+ "num_tokens": 2581170.0,
+ "reward": 0.4875798225402832,
+ "reward_std": 0.16742677986621857,
+ "rewards/true_env_reward_fn/mean": 0.4875798225402832,
+ "rewards/true_env_reward_fn/std": 0.16742677986621857,
+ "step": 105,
+ "step_time": 9.588520330999927
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2687023878097534,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08327006548643112,
+ "kl": 2.3203040655062068e-05,
+ "learning_rate": 9.275938977018081e-07,
+ "loss": 0.003695126622915268,
+ "num_tokens": 2609408.0,
+ "reward": 0.40928101539611816,
+ "reward_std": 0.10633077472448349,
+ "rewards/true_env_reward_fn/mean": 0.40928101539611816,
+ "rewards/true_env_reward_fn/std": 0.1063307598233223,
+ "step": 106,
+ "step_time": 14.60399662399982
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 79.79167175292969,
+ "completions/mean_terminated_length": 79.79167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.1623006761074066,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07009758800268173,
+ "kl": 2.6010310648416635e-05,
+ "learning_rate": 9.253719980342134e-07,
+ "loss": -0.025412028655409813,
+ "num_tokens": 2641446.0,
+ "reward": 0.29606160521507263,
+ "reward_std": 0.3615049123764038,
+ "rewards/true_env_reward_fn/mean": 0.29606160521507263,
+ "rewards/true_env_reward_fn/std": 0.3615049123764038,
+ "step": 107,
+ "step_time": 20.100954443999854
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 60.6875,
+ "completions/mean_terminated_length": 60.6875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2318958044052124,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09370094537734985,
+ "kl": 2.169116805816884e-05,
+ "learning_rate": 9.23119265543822e-07,
+ "loss": -0.009763844311237335,
+ "num_tokens": 2659695.0,
+ "reward": 0.5309837460517883,
+ "reward_std": 0.1692933589220047,
+ "rewards/true_env_reward_fn/mean": 0.5309837460517883,
+ "rewards/true_env_reward_fn/std": 0.1692933589220047,
+ "step": 108,
+ "step_time": 8.304149297999857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 66.29167175292969,
+ "completions/mean_terminated_length": 66.29167175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.248624861240387,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09214548021554947,
+ "kl": 5.0202284000988584e-05,
+ "learning_rate": 9.208358635185372e-07,
+ "loss": 0.0672653466463089,
+ "num_tokens": 2691005.0,
+ "reward": 0.306609183549881,
+ "reward_std": 0.24702024459838867,
+ "rewards/true_env_reward_fn/mean": 0.306609183549881,
+ "rewards/true_env_reward_fn/std": 0.24702024459838867,
+ "step": 109,
+ "step_time": 11.260021517000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 77.64583587646484,
+ "completions/mean_terminated_length": 77.64583587646484,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.2180723249912262,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08635839074850082,
+ "kl": 3.741631462617079e-05,
+ "learning_rate": 9.185219574693241e-07,
+ "loss": 0.06331576406955719,
+ "num_tokens": 2717196.0,
+ "reward": 0.5423221588134766,
+ "reward_std": 0.3347312808036804,
+ "rewards/true_env_reward_fn/mean": 0.5423220992088318,
+ "rewards/true_env_reward_fn/std": 0.3347312808036804,
+ "step": 110,
+ "step_time": 22.80178854000019
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2095272839069366,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09482823312282562,
+ "kl": 3.445757738518296e-05,
+ "learning_rate": 9.161777151182135e-07,
+ "loss": -0.007809684611856937,
+ "num_tokens": 2739924.0,
+ "reward": 0.4738404154777527,
+ "reward_std": 0.2762244939804077,
+ "rewards/true_env_reward_fn/mean": 0.4738403856754303,
+ "rewards/true_env_reward_fn/std": 0.2762244939804077,
+ "step": 111,
+ "step_time": 9.663163859000178
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.79167175292969,
+ "completions/mean_terminated_length": 72.79167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.339354246854782,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09729615598917007,
+ "kl": 4.7237631861207774e-05,
+ "learning_rate": 9.138033063861434e-07,
+ "loss": 0.0440620519220829,
+ "num_tokens": 2763226.0,
+ "reward": 0.4624016284942627,
+ "reward_std": 0.2299472838640213,
+ "rewards/true_env_reward_fn/mean": 0.4624016284942627,
+ "rewards/true_env_reward_fn/std": 0.2299472540616989,
+ "step": 112,
+ "step_time": 9.903081222999617
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 62.5625,
+ "completions/mean_terminated_length": 62.5625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2139239311218262,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09817806631326675,
+ "kl": 5.5064369917090517e-05,
+ "learning_rate": 9.113989033806433e-07,
+ "loss": 0.03889988735318184,
+ "num_tokens": 2788677.0,
+ "reward": 0.3767920434474945,
+ "reward_std": 0.3360261619091034,
+ "rewards/true_env_reward_fn/mean": 0.3767920434474945,
+ "rewards/true_env_reward_fn/std": 0.3360261619091034,
+ "step": 113,
+ "step_time": 10.101770388999284
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 236.0,
+ "completions/max_terminated_length": 236.0,
+ "completions/mean_length": 69.29167175292969,
+ "completions/mean_terminated_length": 69.29167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.2278488278388977,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09011705964803696,
+ "kl": 3.285038519607042e-05,
+ "learning_rate": 9.089646803833588e-07,
+ "loss": 0.03598163276910782,
+ "num_tokens": 2812139.0,
+ "reward": 0.5151569843292236,
+ "reward_std": 0.24896851181983948,
+ "rewards/true_env_reward_fn/mean": 0.5151569247245789,
+ "rewards/true_env_reward_fn/std": 0.24896851181983948,
+ "step": 114,
+ "step_time": 17.633509853000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.97916793823242,
+ "completions/mean_terminated_length": 61.97916793823242,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1842038929462433,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07707802206277847,
+ "kl": 4.165519931120798e-05,
+ "learning_rate": 9.065008138374188e-07,
+ "loss": 0.03350803256034851,
+ "num_tokens": 2835354.0,
+ "reward": 0.4122808873653412,
+ "reward_std": 0.27231934666633606,
+ "rewards/true_env_reward_fn/mean": 0.4122808873653412,
+ "rewards/true_env_reward_fn/std": 0.27231931686401367,
+ "step": 115,
+ "step_time": 12.307247350000125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 232.0,
+ "completions/max_terminated_length": 232.0,
+ "completions/mean_length": 83.64583587646484,
+ "completions/mean_terminated_length": 83.64583587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3357974886894226,
+ "epoch": 2.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06329861283302307,
+ "kl": 1.8487026636648807e-05,
+ "learning_rate": 9.040074823346464e-07,
+ "loss": 0.030132077634334564,
+ "num_tokens": 2859017.0,
+ "reward": 0.5723411440849304,
+ "reward_std": 0.21183526515960693,
+ "rewards/true_env_reward_fn/mean": 0.5723411440849304,
+ "rewards/true_env_reward_fn/std": 0.21183528006076813,
+ "step": 116,
+ "step_time": 14.468690254000194
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.35417175292969,
+ "completions/mean_terminated_length": 70.35417175292969,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.1871840357780457,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06702237576246262,
+ "kl": 2.817388576659141e-05,
+ "learning_rate": 9.014848666026138e-07,
+ "loss": 0.00200769305229187,
+ "num_tokens": 2889050.0,
+ "reward": 0.3500348925590515,
+ "reward_std": 0.30559059977531433,
+ "rewards/true_env_reward_fn/mean": 0.3500348627567291,
+ "rewards/true_env_reward_fn/std": 0.3055906295776367,
+ "step": 117,
+ "step_time": 11.849063975999798
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 79.4375,
+ "completions/mean_terminated_length": 79.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2978005111217499,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0668371245265007,
+ "kl": 3.733048197318567e-05,
+ "learning_rate": 8.989331494915416e-07,
+ "loss": -0.04680684953927994,
+ "num_tokens": 2917335.0,
+ "reward": 0.32068905234336853,
+ "reward_std": 0.30586519837379456,
+ "rewards/true_env_reward_fn/mean": 0.32068905234336853,
+ "rewards/true_env_reward_fn/std": 0.30586519837379456,
+ "step": 118,
+ "step_time": 16.597334930000216
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2123413980007172,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.074281245470047,
+ "kl": 3.654057309177006e-05,
+ "learning_rate": 8.963525159610464e-07,
+ "loss": 0.0373641662299633,
+ "num_tokens": 2938004.0,
+ "reward": 0.556015133857727,
+ "reward_std": 0.22209766507148743,
+ "rewards/true_env_reward_fn/mean": 0.5560150742530823,
+ "rewards/true_env_reward_fn/std": 0.22209767997264862,
+ "step": 119,
+ "step_time": 11.729475523999554
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 66.08333587646484,
+ "completions/mean_terminated_length": 66.08333587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3360244035720825,
+ "epoch": 2.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09382818639278412,
+ "kl": 3.540705620252993e-05,
+ "learning_rate": 8.937431530667327e-07,
+ "loss": 0.057918041944503784,
+ "num_tokens": 2966976.0,
+ "reward": 0.3999954164028168,
+ "reward_std": 0.2351321578025818,
+ "rewards/true_env_reward_fn/mean": 0.3999954164028168,
+ "rewards/true_env_reward_fn/std": 0.23513217270374298,
+ "step": 120,
+ "step_time": 11.503627788000358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 58.97916793823242,
+ "completions/mean_terminated_length": 58.97916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2053601145744324,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07024823874235153,
+ "kl": 3.6033439755556174e-05,
+ "learning_rate": 8.911052499466356e-07,
+ "loss": 0.04910843074321747,
+ "num_tokens": 2987391.0,
+ "reward": 0.5365906953811646,
+ "reward_std": 0.19872017204761505,
+ "rewards/true_env_reward_fn/mean": 0.5365906357765198,
+ "rewards/true_env_reward_fn/std": 0.19872015714645386,
+ "step": 121,
+ "step_time": 8.728293746000418
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 219.0,
+ "completions/max_terminated_length": 219.0,
+ "completions/mean_length": 70.8125,
+ "completions/mean_terminated_length": 70.8125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.296659678220749,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0607762485742569,
+ "kl": 3.175417168677086e-05,
+ "learning_rate": 8.884389978075097e-07,
+ "loss": -0.040265124291181564,
+ "num_tokens": 3009358.0,
+ "reward": 0.49613699316978455,
+ "reward_std": 0.2080756276845932,
+ "rewards/true_env_reward_fn/mean": 0.49613699316978455,
+ "rewards/true_env_reward_fn/std": 0.2080756276845932,
+ "step": 122,
+ "step_time": 15.51957702100026
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 59.4375,
+ "completions/mean_terminated_length": 59.4375,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2992768585681915,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09044087678194046,
+ "kl": 6.319149179034866e-05,
+ "learning_rate": 8.857445899109715e-07,
+ "loss": -0.030733143910765648,
+ "num_tokens": 3035563.0,
+ "reward": 0.34821078181266785,
+ "reward_std": 0.2354777753353119,
+ "rewards/true_env_reward_fn/mean": 0.34821078181266785,
+ "rewards/true_env_reward_fn/std": 0.23547779023647308,
+ "step": 123,
+ "step_time": 8.471463828000196
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 69.64583587646484,
+ "completions/mean_terminated_length": 69.64583587646484,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2003771364688873,
+ "epoch": 3.024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08278124779462814,
+ "kl": 2.7146410047862446e-05,
+ "learning_rate": 8.83022221559489e-07,
+ "loss": 0.02903313934803009,
+ "num_tokens": 3056090.0,
+ "reward": 0.5313810110092163,
+ "reward_std": 0.18604923784732819,
+ "rewards/true_env_reward_fn/mean": 0.5313810110092163,
+ "rewards/true_env_reward_fn/std": 0.18604923784732819,
+ "step": 124,
+ "step_time": 13.438758649999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2338614165782928,
+ "epoch": 3.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06770245730876923,
+ "kl": 2.570231345089269e-05,
+ "learning_rate": 8.802720900822269e-07,
+ "loss": 0.0324365571141243,
+ "num_tokens": 3080424.0,
+ "reward": 0.44920405745506287,
+ "reward_std": 0.206027552485466,
+ "rewards/true_env_reward_fn/mean": 0.44920405745506287,
+ "rewards/true_env_reward_fn/std": 0.2060275673866272,
+ "step": 125,
+ "step_time": 8.654177170999901
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 70.10417175292969,
+ "completions/mean_terminated_length": 70.10417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1446799635887146,
+ "epoch": 3.073170731707317,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060568179935216904,
+ "kl": 3.4001183394138934e-05,
+ "learning_rate": 8.774943948207425e-07,
+ "loss": -0.009533079341053963,
+ "num_tokens": 3100469.0,
+ "reward": 0.5536229610443115,
+ "reward_std": 0.29822590947151184,
+ "rewards/true_env_reward_fn/mean": 0.5536229610443115,
+ "rewards/true_env_reward_fn/std": 0.29822590947151184,
+ "step": 126,
+ "step_time": 10.513378469000145
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 75.5,
+ "completions/mean_terminated_length": 75.5,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3296749591827393,
+ "epoch": 3.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06767670065164566,
+ "kl": 4.0856727537175175e-05,
+ "learning_rate": 8.746893371145365e-07,
+ "loss": -0.023851895704865456,
+ "num_tokens": 3127557.0,
+ "reward": 0.3543795943260193,
+ "reward_std": 0.3506966233253479,
+ "rewards/true_env_reward_fn/mean": 0.3543795645236969,
+ "rewards/true_env_reward_fn/std": 0.3506965935230255,
+ "step": 127,
+ "step_time": 23.20779430600078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.369004338979721,
+ "epoch": 3.1219512195121952,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08670635521411896,
+ "kl": 3.454186935414327e-05,
+ "learning_rate": 8.718571202864597e-07,
+ "loss": 0.03175315260887146,
+ "num_tokens": 3154478.0,
+ "reward": 0.37598031759262085,
+ "reward_std": 0.32647329568862915,
+ "rewards/true_env_reward_fn/mean": 0.37598028779029846,
+ "rewards/true_env_reward_fn/std": 0.32647326588630676,
+ "step": 128,
+ "step_time": 11.551069149999876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2507834732532501,
+ "epoch": 3.1463414634146343,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07138162106275558,
+ "kl": 3.6777758396056015e-05,
+ "learning_rate": 8.689979496279746e-07,
+ "loss": 0.001895703375339508,
+ "num_tokens": 3182339.0,
+ "reward": 0.3563499450683594,
+ "reward_std": 0.2783089876174927,
+ "rewards/true_env_reward_fn/mean": 0.3563499450683594,
+ "rewards/true_env_reward_fn/std": 0.2783089876174927,
+ "step": 129,
+ "step_time": 10.723005456000465
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 71.3125,
+ "completions/mean_terminated_length": 71.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3551637530326843,
+ "epoch": 3.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0931132510304451,
+ "kl": 6.375309385475703e-05,
+ "learning_rate": 8.66112032384275e-07,
+ "loss": -0.04869828745722771,
+ "num_tokens": 3211594.0,
+ "reward": 0.35241150856018066,
+ "reward_std": 0.2379828542470932,
+ "rewards/true_env_reward_fn/mean": 0.35241150856018066,
+ "rewards/true_env_reward_fn/std": 0.2379828542470932,
+ "step": 130,
+ "step_time": 15.15810051499966
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 199.0,
+ "completions/max_terminated_length": 199.0,
+ "completions/mean_length": 66.6875,
+ "completions/mean_terminated_length": 66.6875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.212640792131424,
+ "epoch": 3.1951219512195124,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0767395943403244,
+ "kl": 4.670183352573076e-05,
+ "learning_rate": 8.631995777392644e-07,
+ "loss": -0.02368815243244171,
+ "num_tokens": 3242883.0,
+ "reward": 0.3383604884147644,
+ "reward_std": 0.31325310468673706,
+ "rewards/true_env_reward_fn/mean": 0.338360458612442,
+ "rewards/true_env_reward_fn/std": 0.31325310468673706,
+ "step": 131,
+ "step_time": 20.109428818001106
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 59.520835876464844,
+ "completions/mean_terminated_length": 59.520835876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3366018533706665,
+ "epoch": 3.2195121951219514,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.11203660070896149,
+ "kl": 6.134294108051108e-05,
+ "learning_rate": 8.602607968003934e-07,
+ "loss": -0.03865987807512283,
+ "num_tokens": 3268372.0,
+ "reward": 0.440601110458374,
+ "reward_std": 0.336189866065979,
+ "rewards/true_env_reward_fn/mean": 0.440601110458374,
+ "rewards/true_env_reward_fn/std": 0.336189866065979,
+ "step": 132,
+ "step_time": 10.12403799699996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2847907543182373,
+ "epoch": 3.2439024390243905,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10071831941604614,
+ "kl": 6.808681609982159e-05,
+ "learning_rate": 8.572959025833573e-07,
+ "loss": 0.0024422593414783478,
+ "num_tokens": 3291888.0,
+ "reward": 0.3618060350418091,
+ "reward_std": 0.26743030548095703,
+ "rewards/true_env_reward_fn/mean": 0.3618060350418091,
+ "rewards/true_env_reward_fn/std": 0.26743027567863464,
+ "step": 133,
+ "step_time": 10.396350653999434
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.4375,
+ "completions/mean_terminated_length": 67.4375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1504567563533783,
+ "epoch": 3.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060020897537469864,
+ "kl": 3.462390031927498e-05,
+ "learning_rate": 8.543051099966557e-07,
+ "loss": 0.04882139340043068,
+ "num_tokens": 3317125.0,
+ "reward": 0.5031180381774902,
+ "reward_std": 0.2628377676010132,
+ "rewards/true_env_reward_fn/mean": 0.5031179785728455,
+ "rewards/true_env_reward_fn/std": 0.2628377676010132,
+ "step": 134,
+ "step_time": 10.961974539000039
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 65.45833587646484,
+ "completions/mean_terminated_length": 65.45833587646484,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.3224314153194427,
+ "epoch": 3.292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10321197658777237,
+ "kl": 6.988596032897476e-05,
+ "learning_rate": 8.51288635826016e-07,
+ "loss": 0.011862488463521004,
+ "num_tokens": 3347059.0,
+ "reward": 0.39905214309692383,
+ "reward_std": 0.31803515553474426,
+ "rewards/true_env_reward_fn/mean": 0.39905214309692383,
+ "rewards/true_env_reward_fn/std": 0.31803515553474426,
+ "step": 135,
+ "step_time": 11.779171687000144
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1519232094287872,
+ "epoch": 3.317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.085839182138443,
+ "kl": 5.779342154710321e-05,
+ "learning_rate": 8.482466987186785e-07,
+ "loss": 0.05022352561354637,
+ "num_tokens": 3370225.0,
+ "reward": 0.4742569923400879,
+ "reward_std": 0.3171122074127197,
+ "rewards/true_env_reward_fn/mean": 0.4742569923400879,
+ "rewards/true_env_reward_fn/std": 0.3171122074127197,
+ "step": 136,
+ "step_time": 8.779588141000204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.6875,
+ "completions/mean_terminated_length": 65.6875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.377644658088684,
+ "epoch": 3.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05921673774719238,
+ "kl": 3.667381497507449e-05,
+ "learning_rate": 8.451795191675487e-07,
+ "loss": 0.020366013050079346,
+ "num_tokens": 3399578.0,
+ "reward": 0.3279460370540619,
+ "reward_std": 0.4147447645664215,
+ "rewards/true_env_reward_fn/mean": 0.3279460370540619,
+ "rewards/true_env_reward_fn/std": 0.4147447645664215,
+ "step": 137,
+ "step_time": 11.74765996799988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.583335876464844,
+ "completions/mean_terminated_length": 61.583335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2057753205299377,
+ "epoch": 3.3658536585365852,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08148445934057236,
+ "kl": 3.0601177968492266e-05,
+ "learning_rate": 8.420873194952152e-07,
+ "loss": 0.005453992635011673,
+ "num_tokens": 3417734.0,
+ "reward": 0.5946073532104492,
+ "reward_std": 0.25090643763542175,
+ "rewards/true_env_reward_fn/mean": 0.5946073532104492,
+ "rewards/true_env_reward_fn/std": 0.25090643763542175,
+ "step": 138,
+ "step_time": 8.8135579650002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3243012130260468,
+ "epoch": 3.3902439024390243,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06626639515161514,
+ "kl": 3.7586045436910354e-05,
+ "learning_rate": 8.389703238378338e-07,
+ "loss": -0.003325441852211952,
+ "num_tokens": 3441749.0,
+ "reward": 0.48056626319885254,
+ "reward_std": 0.2497076690196991,
+ "rewards/true_env_reward_fn/mean": 0.48056626319885254,
+ "rewards/true_env_reward_fn/std": 0.2497076541185379,
+ "step": 139,
+ "step_time": 9.997661417000472
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 75.79167175292969,
+ "completions/mean_terminated_length": 75.79167175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918364107608795,
+ "epoch": 3.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0600166842341423,
+ "kl": 3.408677366678603e-05,
+ "learning_rate": 8.358287581288822e-07,
+ "loss": -0.002709554508328438,
+ "num_tokens": 3473139.0,
+ "reward": 0.38171443343162537,
+ "reward_std": 0.2058144509792328,
+ "rewards/true_env_reward_fn/mean": 0.38171443343162537,
+ "rewards/true_env_reward_fn/std": 0.2058144509792328,
+ "step": 140,
+ "step_time": 14.679971276999822
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.6875,
+ "completions/mean_terminated_length": 62.6875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.32420814037323,
+ "epoch": 3.4390243902439024,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08056586235761642,
+ "kl": 4.234552670823177e-05,
+ "learning_rate": 8.326628500827825e-07,
+ "loss": 0.019535928964614868,
+ "num_tokens": 3499324.0,
+ "reward": 0.4587298631668091,
+ "reward_std": 0.3119663596153259,
+ "rewards/true_env_reward_fn/mean": 0.4587298631668091,
+ "rewards/true_env_reward_fn/std": 0.3119663596153259,
+ "step": 141,
+ "step_time": 11.64747691499997
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.27083587646484,
+ "completions/mean_terminated_length": 69.27083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3127666413784027,
+ "epoch": 3.4634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07890015095472336,
+ "kl": 4.281656902094255e-05,
+ "learning_rate": 8.294728291783965e-07,
+ "loss": -0.034988921135663986,
+ "num_tokens": 3516425.0,
+ "reward": 0.6331583261489868,
+ "reward_std": 0.2317410111427307,
+ "rewards/true_env_reward_fn/mean": 0.633158266544342,
+ "rewards/true_env_reward_fn/std": 0.23174098134040833,
+ "step": 142,
+ "step_time": 8.380270293999729
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 73.97917175292969,
+ "completions/mean_terminated_length": 73.97917175292969,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.323029786348343,
+ "epoch": 3.4878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08455090969800949,
+ "kl": 6.877856139908545e-05,
+ "learning_rate": 8.262589266423908e-07,
+ "loss": 0.06993371993303299,
+ "num_tokens": 3542912.0,
+ "reward": 0.41727983951568604,
+ "reward_std": 0.23754946887493134,
+ "rewards/true_env_reward_fn/mean": 0.41727983951568604,
+ "rewards/true_env_reward_fn/std": 0.23754946887493134,
+ "step": 143,
+ "step_time": 11.716556537000088
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 63.3125,
+ "completions/mean_terminated_length": 63.3125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3465435802936554,
+ "epoch": 3.5121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09822116792201996,
+ "kl": 8.00468278612243e-05,
+ "learning_rate": 8.230213754324772e-07,
+ "loss": 0.07691670209169388,
+ "num_tokens": 3569575.0,
+ "reward": 0.28445714712142944,
+ "reward_std": 0.33810389041900635,
+ "rewards/true_env_reward_fn/mean": 0.28445711731910706,
+ "rewards/true_env_reward_fn/std": 0.33810392022132874,
+ "step": 144,
+ "step_time": 10.67718802499985
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3208706378936768,
+ "epoch": 3.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776741147041321,
+ "kl": 4.5862597744417144e-05,
+ "learning_rate": 8.19760410220527e-07,
+ "loss": -0.014808554202318192,
+ "num_tokens": 3589641.0,
+ "reward": 0.5829761028289795,
+ "reward_std": 0.21224236488342285,
+ "rewards/true_env_reward_fn/mean": 0.5829761028289795,
+ "rewards/true_env_reward_fn/std": 0.21224237978458405,
+ "step": 145,
+ "step_time": 8.610043666999445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2934723794460297,
+ "epoch": 3.5609756097560976,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05893269553780556,
+ "kl": 3.648432630143361e-05,
+ "learning_rate": 8.164762673755609e-07,
+ "loss": 0.023374930024147034,
+ "num_tokens": 3615570.0,
+ "reward": 0.47375163435935974,
+ "reward_std": 0.16054874658584595,
+ "rewards/true_env_reward_fn/mean": 0.47375163435935974,
+ "rewards/true_env_reward_fn/std": 0.16054873168468475,
+ "step": 146,
+ "step_time": 13.649344002000362
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 72.1875,
+ "completions/mean_terminated_length": 72.1875,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3180726766586304,
+ "epoch": 3.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08518138527870178,
+ "kl": 6.788871905882843e-05,
+ "learning_rate": 8.131691849466152e-07,
+ "loss": -0.04987313598394394,
+ "num_tokens": 3637475.0,
+ "reward": 0.5195532441139221,
+ "reward_std": 0.26043611764907837,
+ "rewards/true_env_reward_fn/mean": 0.5195532441139221,
+ "rewards/true_env_reward_fn/std": 0.26043611764907837,
+ "step": 147,
+ "step_time": 11.702765863000877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.83333587646484,
+ "completions/mean_terminated_length": 73.83333587646484,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.167496383190155,
+ "epoch": 3.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07978484779596329,
+ "kl": 6.430712710425723e-05,
+ "learning_rate": 8.098394026454884e-07,
+ "loss": 0.024383332580327988,
+ "num_tokens": 3663171.0,
+ "reward": 0.4524516761302948,
+ "reward_std": 0.2587544322013855,
+ "rewards/true_env_reward_fn/mean": 0.4524516761302948,
+ "rewards/true_env_reward_fn/std": 0.2587544322013855,
+ "step": 148,
+ "step_time": 13.306644664000487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.155810385942459,
+ "epoch": 3.6341463414634148,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046879056841135025,
+ "kl": 4.4023097871104255e-05,
+ "learning_rate": 8.064871618293645e-07,
+ "loss": -0.01477135717868805,
+ "num_tokens": 3683813.0,
+ "reward": 0.6097190380096436,
+ "reward_std": 0.17910261452198029,
+ "rewards/true_env_reward_fn/mean": 0.6097190380096436,
+ "rewards/true_env_reward_fn/std": 0.17910261452198029,
+ "step": 149,
+ "step_time": 9.446422488999815
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.8125,
+ "completions/mean_terminated_length": 72.8125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3542745113372803,
+ "epoch": 3.658536585365854,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06843585520982742,
+ "kl": 3.291011944384081e-05,
+ "learning_rate": 8.03112705483319e-07,
+ "loss": 0.009258950129151344,
+ "num_tokens": 3702516.0,
+ "reward": 0.5345131754875183,
+ "reward_std": 0.22612926363945007,
+ "rewards/true_env_reward_fn/mean": 0.5345131754875183,
+ "rewards/true_env_reward_fn/std": 0.22612926363945007,
+ "step": 150,
+ "step_time": 10.538116119999813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 66.58333587646484,
+ "completions/mean_terminated_length": 66.58333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3427793979644775,
+ "epoch": 3.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0966033861041069,
+ "kl": 7.367974285443779e-05,
+ "learning_rate": 7.99716278202706e-07,
+ "loss": 0.07012784481048584,
+ "num_tokens": 3733800.0,
+ "reward": 0.3090733289718628,
+ "reward_std": 0.3846965730190277,
+ "rewards/true_env_reward_fn/mean": 0.3090732991695404,
+ "rewards/true_env_reward_fn/std": 0.3846965730190277,
+ "step": 151,
+ "step_time": 15.755764130999978
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3039455115795135,
+ "epoch": 3.7073170731707314,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06908538937568665,
+ "kl": 4.5496621623897227e-05,
+ "learning_rate": 7.962981261754294e-07,
+ "loss": 0.02471787855029106,
+ "num_tokens": 3758710.0,
+ "reward": 0.35497602820396423,
+ "reward_std": 0.25890877842903137,
+ "rewards/true_env_reward_fn/mean": 0.35497602820396423,
+ "rewards/true_env_reward_fn/std": 0.25890880823135376,
+ "step": 152,
+ "step_time": 9.670861957999932
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 369.0,
+ "completions/max_terminated_length": 369.0,
+ "completions/mean_length": 71.75,
+ "completions/mean_terminated_length": 71.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.342434972524643,
+ "epoch": 3.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0625183954834938,
+ "kl": 5.662065541400807e-05,
+ "learning_rate": 7.928584971640974e-07,
+ "loss": 0.15159915387630463,
+ "num_tokens": 3781818.0,
+ "reward": 0.456516832113266,
+ "reward_std": 0.291423499584198,
+ "rewards/true_env_reward_fn/mean": 0.456516832113266,
+ "rewards/true_env_reward_fn/std": 0.291423499584198,
+ "step": 153,
+ "step_time": 22.82054339000024
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.256364792585373,
+ "epoch": 3.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0713125616312027,
+ "kl": 5.741999029851286e-05,
+ "learning_rate": 7.893976404880641e-07,
+ "loss": 0.0011316314339637756,
+ "num_tokens": 3801434.0,
+ "reward": 0.6220619082450867,
+ "reward_std": 0.260771244764328,
+ "rewards/true_env_reward_fn/mean": 0.6220619082450867,
+ "rewards/true_env_reward_fn/std": 0.260771244764328,
+ "step": 154,
+ "step_time": 9.973958625999785
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 75.02083587646484,
+ "completions/mean_terminated_length": 75.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3467005491256714,
+ "epoch": 3.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05885510891675949,
+ "kl": 5.8398384226165945e-05,
+ "learning_rate": 7.859158070053576e-07,
+ "loss": -0.006662093102931976,
+ "num_tokens": 3829875.0,
+ "reward": 0.4248020052909851,
+ "reward_std": 0.17955487966537476,
+ "rewards/true_env_reward_fn/mean": 0.4248019754886627,
+ "rewards/true_env_reward_fn/std": 0.17955489456653595,
+ "step": 155,
+ "step_time": 9.758407419000378
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2418483197689056,
+ "epoch": 3.8048780487804876,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07287121564149857,
+ "kl": 7.330268545047147e-05,
+ "learning_rate": 7.824132490944967e-07,
+ "loss": 0.009832290932536125,
+ "num_tokens": 3858478.0,
+ "reward": 0.45385628938674927,
+ "reward_std": 0.241779163479805,
+ "rewards/true_env_reward_fn/mean": 0.4538562595844269,
+ "rewards/true_env_reward_fn/std": 0.2417791783809662,
+ "step": 156,
+ "step_time": 11.009583763999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2394451797008514,
+ "epoch": 3.8292682926829267,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07873199880123138,
+ "kl": 3.887376169586787e-05,
+ "learning_rate": 7.788902206361973e-07,
+ "loss": 0.004030962474644184,
+ "num_tokens": 3882682.0,
+ "reward": 0.5042052268981934,
+ "reward_std": 0.17870797216892242,
+ "rewards/true_env_reward_fn/mean": 0.5042052268981934,
+ "rewards/true_env_reward_fn/std": 0.17870797216892242,
+ "step": 157,
+ "step_time": 10.12789283499933
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 60.458335876464844,
+ "completions/mean_terminated_length": 60.458335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.148638516664505,
+ "epoch": 3.8536585365853657,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06768295168876648,
+ "kl": 5.1569048991950694e-05,
+ "learning_rate": 7.7534697699497e-07,
+ "loss": -0.018120331689715385,
+ "num_tokens": 3902176.0,
+ "reward": 0.5385247468948364,
+ "reward_std": 0.20308326184749603,
+ "rewards/true_env_reward_fn/mean": 0.5385246872901917,
+ "rewards/true_env_reward_fn/std": 0.20308324694633484,
+ "step": 158,
+ "step_time": 7.800485663000472
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0718395709991455,
+ "epoch": 3.8780487804878048,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08181443065404892,
+ "kl": 6.815949382144026e-05,
+ "learning_rate": 7.717837750006106e-07,
+ "loss": -0.007854004390537739,
+ "num_tokens": 3930658.0,
+ "reward": 0.4394054412841797,
+ "reward_std": 0.3746899962425232,
+ "rewards/true_env_reward_fn/mean": 0.4394054412841797,
+ "rewards/true_env_reward_fn/std": 0.3746899962425232,
+ "step": 159,
+ "step_time": 13.77649076500029
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 65.33333587646484,
+ "completions/mean_terminated_length": 65.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.135006695985794,
+ "epoch": 3.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07245675474405289,
+ "kl": 6.525267690449255e-05,
+ "learning_rate": 7.682008729295833e-07,
+ "loss": 0.07331673055887222,
+ "num_tokens": 3958082.0,
+ "reward": 0.3795818090438843,
+ "reward_std": 0.21483220160007477,
+ "rewards/true_env_reward_fn/mean": 0.3795818090438843,
+ "rewards/true_env_reward_fn/std": 0.21483221650123596,
+ "step": 160,
+ "step_time": 13.25029361300085
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 286.0,
+ "completions/max_terminated_length": 286.0,
+ "completions/mean_length": 80.66667175292969,
+ "completions/mean_terminated_length": 80.66667175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.242073804140091,
+ "epoch": 3.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06932233273983002,
+ "kl": 6.277856755332323e-05,
+ "learning_rate": 7.645985304863003e-07,
+ "loss": 0.05312003195285797,
+ "num_tokens": 3984066.0,
+ "reward": 0.4469220042228699,
+ "reward_std": 0.17845165729522705,
+ "rewards/true_env_reward_fn/mean": 0.4469219744205475,
+ "rewards/true_env_reward_fn/std": 0.17845165729522705,
+ "step": 161,
+ "step_time": 17.837881629000094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 52.145835876464844,
+ "completions/mean_terminated_length": 52.145835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1892985105514526,
+ "epoch": 3.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10256654024124146,
+ "kl": 7.12887790541572e-05,
+ "learning_rate": 7.609770087842968e-07,
+ "loss": -0.046506255865097046,
+ "num_tokens": 4003625.0,
+ "reward": 0.49098464846611023,
+ "reward_std": 0.3060121238231659,
+ "rewards/true_env_reward_fn/mean": 0.49098464846611023,
+ "rewards/true_env_reward_fn/std": 0.3060121238231659,
+ "step": 162,
+ "step_time": 9.286757633000889
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1780613362789154,
+ "epoch": 3.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07468484342098236,
+ "kl": 5.8644353430281626e-05,
+ "learning_rate": 7.573365703273045e-07,
+ "loss": -0.0016099847853183746,
+ "num_tokens": 4024676.0,
+ "reward": 0.5258157253265381,
+ "reward_std": 0.2065279185771942,
+ "rewards/true_env_reward_fn/mean": 0.5258157253265381,
+ "rewards/true_env_reward_fn/std": 0.2065279185771942,
+ "step": 163,
+ "step_time": 10.785562561999996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 67.20833587646484,
+ "completions/mean_terminated_length": 67.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1701751053333282,
+ "epoch": 4.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06985389441251755,
+ "kl": 3.805391224886989e-05,
+ "learning_rate": 7.536774789902245e-07,
+ "loss": 0.0351775586605072,
+ "num_tokens": 4048798.0,
+ "reward": 0.520969033241272,
+ "reward_std": 0.118266262114048,
+ "rewards/true_env_reward_fn/mean": 0.520969033241272,
+ "rewards/true_env_reward_fn/std": 0.1182662770152092,
+ "step": 164,
+ "step_time": 12.722446307000155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 202.0,
+ "completions/max_terminated_length": 202.0,
+ "completions/mean_length": 62.16666793823242,
+ "completions/mean_terminated_length": 62.16666793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1948959231376648,
+ "epoch": 4.024390243902439,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07691483944654465,
+ "kl": 6.559857092724997e-05,
+ "learning_rate": 7.5e-07,
+ "loss": 0.010169023647904396,
+ "num_tokens": 4076302.0,
+ "reward": 0.3847707509994507,
+ "reward_std": 0.2532062828540802,
+ "rewards/true_env_reward_fn/mean": 0.3847707509994507,
+ "rewards/true_env_reward_fn/std": 0.2532062828540802,
+ "step": 165,
+ "step_time": 14.89655208000022
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 61.645835876464844,
+ "completions/mean_terminated_length": 61.645835876464844,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.176283597946167,
+ "epoch": 4.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08597421646118164,
+ "kl": 7.07068675183109e-05,
+ "learning_rate": 7.463043999163918e-07,
+ "loss": 0.030138514935970306,
+ "num_tokens": 4096853.0,
+ "reward": 0.5679852962493896,
+ "reward_std": 0.23158182203769684,
+ "rewards/true_env_reward_fn/mean": 0.5679852366447449,
+ "rewards/true_env_reward_fn/std": 0.23158180713653564,
+ "step": 166,
+ "step_time": 8.962532588000613
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 69.33333587646484,
+ "completions/mean_terminated_length": 69.33333587646484,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.3250808417797089,
+ "epoch": 4.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08058538287878036,
+ "kl": 6.5122869273182e-05,
+ "learning_rate": 7.425909466126568e-07,
+ "loss": 0.03569657728075981,
+ "num_tokens": 4118253.0,
+ "reward": 0.5882628560066223,
+ "reward_std": 0.2646455764770508,
+ "rewards/true_env_reward_fn/mean": 0.5882628560066223,
+ "rewards/true_env_reward_fn/std": 0.2646455764770508,
+ "step": 167,
+ "step_time": 11.708963983000103
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 261.0,
+ "completions/max_terminated_length": 261.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.319758415222168,
+ "epoch": 4.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07505878806114197,
+ "kl": 4.8000228161981795e-05,
+ "learning_rate": 7.388599092561314e-07,
+ "loss": 0.10015778243541718,
+ "num_tokens": 4142931.0,
+ "reward": 0.5041360259056091,
+ "reward_std": 0.29254013299942017,
+ "rewards/true_env_reward_fn/mean": 0.5041360259056091,
+ "rewards/true_env_reward_fn/std": 0.2925401031970978,
+ "step": 168,
+ "step_time": 16.905241970000134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.333335876464844,
+ "completions/mean_terminated_length": 60.333335876464844,
+ "completions/min_length": 17.0,
+ "completions/min_terminated_length": 17.0,
+ "entropy": 1.3414625525474548,
+ "epoch": 4.121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08060909807682037,
+ "kl": 6.745914834027644e-05,
+ "learning_rate": 7.351115582887211e-07,
+ "loss": -0.04827923700213432,
+ "num_tokens": 4172755.0,
+ "reward": 0.4034843444824219,
+ "reward_std": 0.29997992515563965,
+ "rewards/true_env_reward_fn/mean": 0.4034843444824219,
+ "rewards/true_env_reward_fn/std": 0.29997992515563965,
+ "step": 169,
+ "step_time": 12.27045077799994
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 155.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 63.53191375732422,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.4034385085105896,
+ "epoch": 4.146341463414634,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05382552370429039,
+ "kl": 3.640815248218132e-05,
+ "learning_rate": 7.313461654072973e-07,
+ "loss": 0.07585563510656357,
+ "num_tokens": 4196629.0,
+ "reward": 0.5297740697860718,
+ "reward_std": 0.25283464789390564,
+ "rewards/true_env_reward_fn/mean": 0.529774010181427,
+ "rewards/true_env_reward_fn/std": 0.25283464789390564,
+ "step": 170,
+ "step_time": 31.721865252000498
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2666764855384827,
+ "epoch": 4.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06263021379709244,
+ "kl": 5.531543774850434e-05,
+ "learning_rate": 7.275640035440044e-07,
+ "loss": -0.004293827340006828,
+ "num_tokens": 4215315.0,
+ "reward": 0.5377860069274902,
+ "reward_std": 0.22855143249034882,
+ "rewards/true_env_reward_fn/mean": 0.5377860069274902,
+ "rewards/true_env_reward_fn/std": 0.2285514622926712,
+ "step": 171,
+ "step_time": 8.973740739000277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2872737050056458,
+ "epoch": 4.195121951219512,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09720690548419952,
+ "kl": 8.13291462691268e-05,
+ "learning_rate": 7.237653468464755e-07,
+ "loss": 0.015124019235372543,
+ "num_tokens": 4235707.0,
+ "reward": 0.5968735814094543,
+ "reward_std": 0.2860201299190521,
+ "rewards/true_env_reward_fn/mean": 0.5968735814094543,
+ "rewards/true_env_reward_fn/std": 0.28602010011672974,
+ "step": 172,
+ "step_time": 10.39117795600032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 74.77083587646484,
+ "completions/mean_terminated_length": 74.77083587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2326436638832092,
+ "epoch": 4.219512195121951,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.046250198036432266,
+ "kl": 4.305509810365038e-05,
+ "learning_rate": 7.199504706579616e-07,
+ "loss": -0.010809645056724548,
+ "num_tokens": 4262504.0,
+ "reward": 0.46610039472579956,
+ "reward_std": 0.2549833357334137,
+ "rewards/true_env_reward_fn/mean": 0.4661003649234772,
+ "rewards/true_env_reward_fn/std": 0.2549833059310913,
+ "step": 173,
+ "step_time": 11.8111169230001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 77.16667175292969,
+ "completions/mean_terminated_length": 77.16667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.261723816394806,
+ "epoch": 4.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07238215953111649,
+ "kl": 7.066424404911231e-05,
+ "learning_rate": 7.161196514973734e-07,
+ "loss": 0.04901377111673355,
+ "num_tokens": 4290472.0,
+ "reward": 0.3766266703605652,
+ "reward_std": 0.27605685591697693,
+ "rewards/true_env_reward_fn/mean": 0.3766266405582428,
+ "rewards/true_env_reward_fn/std": 0.27605685591697693,
+ "step": 174,
+ "step_time": 17.211099596999702
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 74.14583587646484,
+ "completions/mean_terminated_length": 74.14583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.284770429134369,
+ "epoch": 4.2682926829268295,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.03949572518467903,
+ "kl": 3.151347550556238e-05,
+ "learning_rate": 7.12273167039238e-07,
+ "loss": 0.007944343611598015,
+ "num_tokens": 4316687.0,
+ "reward": 0.46209168434143066,
+ "reward_std": 0.1050746738910675,
+ "rewards/true_env_reward_fn/mean": 0.46209168434143066,
+ "rewards/true_env_reward_fn/std": 0.1050746738910675,
+ "step": 175,
+ "step_time": 12.29185969800028
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3430605232715607,
+ "epoch": 4.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09476255625486374,
+ "kl": 9.628380030335393e-05,
+ "learning_rate": 7.084112960935715e-07,
+ "loss": -0.006478719413280487,
+ "num_tokens": 4338257.0,
+ "reward": 0.38874804973602295,
+ "reward_std": 0.2893269658088684,
+ "rewards/true_env_reward_fn/mean": 0.38874804973602295,
+ "rewards/true_env_reward_fn/std": 0.289326936006546,
+ "step": 176,
+ "step_time": 10.987576109999736
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 134.0,
+ "completions/max_terminated_length": 134.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3582488000392914,
+ "epoch": 4.317073170731708,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0771971270442009,
+ "kl": 8.3626783634827e-05,
+ "learning_rate": 7.0453431858567e-07,
+ "loss": 0.032796651124954224,
+ "num_tokens": 4362418.0,
+ "reward": 0.4179423749446869,
+ "reward_std": 0.33730608224868774,
+ "rewards/true_env_reward_fn/mean": 0.4179423749446869,
+ "rewards/true_env_reward_fn/std": 0.33730608224868774,
+ "step": 177,
+ "step_time": 10.792315139000493
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 142.0,
+ "completions/max_terminated_length": 142.0,
+ "completions/mean_length": 58.208335876464844,
+ "completions/mean_terminated_length": 58.208335876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1685997247695923,
+ "epoch": 4.341463414634147,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10942906141281128,
+ "kl": 5.940973733231658e-05,
+ "learning_rate": 7.006425155358195e-07,
+ "loss": 0.12245109677314758,
+ "num_tokens": 4381716.0,
+ "reward": 0.5206946134567261,
+ "reward_std": 0.31685587763786316,
+ "rewards/true_env_reward_fn/mean": 0.5206945538520813,
+ "rewards/true_env_reward_fn/std": 0.3168558180332184,
+ "step": 178,
+ "step_time": 12.399353334999887
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 77.6875,
+ "completions/mean_terminated_length": 77.6875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.302725076675415,
+ "epoch": 4.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06826934218406677,
+ "kl": 3.6839799577137455e-05,
+ "learning_rate": 6.967361690389258e-07,
+ "loss": -0.03518716245889664,
+ "num_tokens": 4405845.0,
+ "reward": 0.5313202142715454,
+ "reward_std": 0.18632179498672485,
+ "rewards/true_env_reward_fn/mean": 0.5313201546669006,
+ "rewards/true_env_reward_fn/std": 0.18632179498672485,
+ "step": 179,
+ "step_time": 12.45691162600042
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 73.39583587646484,
+ "completions/mean_terminated_length": 73.39583587646484,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2715371549129486,
+ "epoch": 4.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05825073644518852,
+ "kl": 7.535525219282135e-05,
+ "learning_rate": 6.928155622440679e-07,
+ "loss": -0.018800390884280205,
+ "num_tokens": 4426576.0,
+ "reward": 0.47252464294433594,
+ "reward_std": 0.33459845185279846,
+ "rewards/true_env_reward_fn/mean": 0.47252464294433594,
+ "rewards/true_env_reward_fn/std": 0.33459845185279846,
+ "step": 180,
+ "step_time": 12.108760526000424
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 61.020835876464844,
+ "completions/mean_terminated_length": 61.020835876464844,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1720183491706848,
+ "epoch": 4.414634146341464,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08154451847076416,
+ "kl": 9.67955465966952e-05,
+ "learning_rate": 6.888809793339728e-07,
+ "loss": 0.016710905358195305,
+ "num_tokens": 4448649.0,
+ "reward": 0.5532544851303101,
+ "reward_std": 0.17937251925468445,
+ "rewards/true_env_reward_fn/mean": 0.5532544255256653,
+ "rewards/true_env_reward_fn/std": 0.17937250435352325,
+ "step": 181,
+ "step_time": 14.16400909000049
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 193.0,
+ "completions/max_terminated_length": 193.0,
+ "completions/mean_length": 71.9375,
+ "completions/mean_terminated_length": 71.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2416147291660309,
+ "epoch": 4.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05918826535344124,
+ "kl": 6.67227295707562e-05,
+ "learning_rate": 6.849327055044181e-07,
+ "loss": 0.04629965499043465,
+ "num_tokens": 4479382.0,
+ "reward": 0.3571457862854004,
+ "reward_std": 0.3042747676372528,
+ "rewards/true_env_reward_fn/mean": 0.3571457862854004,
+ "rewards/true_env_reward_fn/std": 0.3042747676372528,
+ "step": 182,
+ "step_time": 19.767916835999586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 70.52083587646484,
+ "completions/mean_terminated_length": 70.52083587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2693078517913818,
+ "epoch": 4.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06269724667072296,
+ "kl": 4.1979371417255607e-05,
+ "learning_rate": 6.809710269435589e-07,
+ "loss": 0.003845077008008957,
+ "num_tokens": 4501263.0,
+ "reward": 0.5583165884017944,
+ "reward_std": 0.19653278589248657,
+ "rewards/true_env_reward_fn/mean": 0.5583165287971497,
+ "rewards/true_env_reward_fn/std": 0.19653277099132538,
+ "step": 183,
+ "step_time": 10.428195530999346
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 183.0,
+ "completions/max_terminated_length": 183.0,
+ "completions/mean_length": 77.66667175292969,
+ "completions/mean_terminated_length": 77.66667175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2504103481769562,
+ "epoch": 4.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07646579295396805,
+ "kl": 9.529235103400424e-05,
+ "learning_rate": 6.769962308111839e-07,
+ "loss": 0.007325906306505203,
+ "num_tokens": 4525959.0,
+ "reward": 0.4377995431423187,
+ "reward_std": 0.330658495426178,
+ "rewards/true_env_reward_fn/mean": 0.4377995431423187,
+ "rewards/true_env_reward_fn/std": 0.330658495426178,
+ "step": 184,
+ "step_time": 16.47171987999991
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 71.97917175292969,
+ "completions/mean_terminated_length": 71.97917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2982739210128784,
+ "epoch": 4.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07417281717061996,
+ "kl": 4.4408230678527616e-05,
+ "learning_rate": 6.730086052179002e-07,
+ "loss": 0.0469856858253479,
+ "num_tokens": 4546310.0,
+ "reward": 0.5151915550231934,
+ "reward_std": 0.22422264516353607,
+ "rewards/true_env_reward_fn/mean": 0.5151915550231934,
+ "rewards/true_env_reward_fn/std": 0.22422264516353607,
+ "step": 185,
+ "step_time": 13.684267182999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 65.70833587646484,
+ "completions/mean_terminated_length": 65.70833587646484,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2992590963840485,
+ "epoch": 4.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07500731945037842,
+ "kl": 6.5705213273759e-05,
+ "learning_rate": 6.690084392042513e-07,
+ "loss": -0.006503798067569733,
+ "num_tokens": 4566816.0,
+ "reward": 0.5268750190734863,
+ "reward_std": 0.23048490285873413,
+ "rewards/true_env_reward_fn/mean": 0.5268749594688416,
+ "rewards/true_env_reward_fn/std": 0.23048490285873413,
+ "step": 186,
+ "step_time": 10.281018189999031
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 355.0,
+ "completions/max_terminated_length": 355.0,
+ "completions/mean_length": 66.91667175292969,
+ "completions/mean_terminated_length": 66.91667175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.3209801018238068,
+ "epoch": 4.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.082595095038414,
+ "kl": 8.261651692009764e-05,
+ "learning_rate": 6.649960227197647e-07,
+ "loss": 0.0495578795671463,
+ "num_tokens": 4589204.0,
+ "reward": 0.4924369752407074,
+ "reward_std": 0.37204882502555847,
+ "rewards/true_env_reward_fn/mean": 0.4924369752407074,
+ "rewards/true_env_reward_fn/std": 0.37204885482788086,
+ "step": 187,
+ "step_time": 24.351223329000277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.9375,
+ "completions/mean_terminated_length": 62.9375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3033248782157898,
+ "epoch": 4.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0885128378868103,
+ "kl": 0.000125904198284843,
+ "learning_rate": 6.609716466019355e-07,
+ "loss": 0.04691624641418457,
+ "num_tokens": 4622361.0,
+ "reward": 0.16021786630153656,
+ "reward_std": 0.24737857282161713,
+ "rewards/true_env_reward_fn/mean": 0.16021786630153656,
+ "rewards/true_env_reward_fn/std": 0.24737857282161713,
+ "step": 188,
+ "step_time": 10.644911742000204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2344954907894135,
+ "epoch": 4.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07845015823841095,
+ "kl": 8.022368183446815e-05,
+ "learning_rate": 6.569356025551454e-07,
+ "loss": 0.03257204219698906,
+ "num_tokens": 4649381.0,
+ "reward": 0.3547590970993042,
+ "reward_std": 0.23717434704303741,
+ "rewards/true_env_reward_fn/mean": 0.3547590970993042,
+ "rewards/true_env_reward_fn/std": 0.23717434704303741,
+ "step": 189,
+ "step_time": 9.579594637999435
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 146.0,
+ "completions/max_terminated_length": 146.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.258386880159378,
+ "epoch": 4.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07976282387971878,
+ "kl": 0.00011880166857736185,
+ "learning_rate": 6.528881831295188e-07,
+ "loss": -0.02127165161073208,
+ "num_tokens": 4674357.0,
+ "reward": 0.4042941927909851,
+ "reward_std": 0.2737519145011902,
+ "rewards/true_env_reward_fn/mean": 0.4042941629886627,
+ "rewards/true_env_reward_fn/std": 0.2737519443035126,
+ "step": 190,
+ "step_time": 11.495368679999501
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2503610253334045,
+ "epoch": 4.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08848423510789871,
+ "kl": 0.00012839957071264507,
+ "learning_rate": 6.488296816997173e-07,
+ "loss": 0.04479096084833145,
+ "num_tokens": 4694425.0,
+ "reward": 0.5075992345809937,
+ "reward_std": 0.2306082397699356,
+ "rewards/true_env_reward_fn/mean": 0.5075991749763489,
+ "rewards/true_env_reward_fn/std": 0.2306082397699356,
+ "step": 191,
+ "step_time": 7.909104242000012
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 60.520835876464844,
+ "completions/mean_terminated_length": 60.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.266694962978363,
+ "epoch": 4.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0782349705696106,
+ "kl": 8.841241015034029e-05,
+ "learning_rate": 6.447603924436743e-07,
+ "loss": 0.030293334275484085,
+ "num_tokens": 4720074.0,
+ "reward": 0.42062053084373474,
+ "reward_std": 0.17757493257522583,
+ "rewards/true_env_reward_fn/mean": 0.42062053084373474,
+ "rewards/true_env_reward_fn/std": 0.17757493257522583,
+ "step": 192,
+ "step_time": 10.043768619000275
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 63.60416793823242,
+ "completions/mean_terminated_length": 63.60416793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.23249152302742,
+ "epoch": 4.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09907371550798416,
+ "kl": 0.0001827988016884774,
+ "learning_rate": 6.406806103212724e-07,
+ "loss": 0.010011034086346626,
+ "num_tokens": 4746719.0,
+ "reward": 0.426013708114624,
+ "reward_std": 0.3213046193122864,
+ "rewards/true_env_reward_fn/mean": 0.426013708114624,
+ "rewards/true_env_reward_fn/std": 0.3213046193122864,
+ "step": 193,
+ "step_time": 10.19648474899941
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 64.0625,
+ "completions/mean_terminated_length": 64.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2059581279754639,
+ "epoch": 4.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08840472996234894,
+ "kl": 0.00012012650222459342,
+ "learning_rate": 6.365906310529629e-07,
+ "loss": -0.034412819892168045,
+ "num_tokens": 4780578.0,
+ "reward": 0.28782567381858826,
+ "reward_std": 0.3653683662414551,
+ "rewards/true_env_reward_fn/mean": 0.28782567381858826,
+ "rewards/true_env_reward_fn/std": 0.3653683662414551,
+ "step": 194,
+ "step_time": 14.372816425000565
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 77.10417175292969,
+ "completions/mean_terminated_length": 77.10417175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3157014548778534,
+ "epoch": 4.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07452306151390076,
+ "kl": 8.27656508590735e-05,
+ "learning_rate": 6.32490751098331e-07,
+ "loss": -0.004008886404335499,
+ "num_tokens": 4809119.0,
+ "reward": 0.4407285451889038,
+ "reward_std": 0.23340703547000885,
+ "rewards/true_env_reward_fn/mean": 0.4407285451889038,
+ "rewards/true_env_reward_fn/std": 0.23340705037117004,
+ "step": 195,
+ "step_time": 11.282298853000157
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.458335876464844,
+ "completions/mean_terminated_length": 61.458335876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2990687191486359,
+ "epoch": 4.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07655440270900726,
+ "kl": 0.0001329305760009447,
+ "learning_rate": 6.283812676346063e-07,
+ "loss": 0.024208612740039825,
+ "num_tokens": 4835557.0,
+ "reward": 0.42621374130249023,
+ "reward_std": 0.28145232796669006,
+ "rewards/true_env_reward_fn/mean": 0.42621374130249023,
+ "rewards/true_env_reward_fn/std": 0.28145232796669006,
+ "step": 196,
+ "step_time": 8.65745804199969
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 56.333335876464844,
+ "completions/mean_terminated_length": 56.333335876464844,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.148613840341568,
+ "epoch": 4.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08951497822999954,
+ "kl": 5.616615226244903e-05,
+ "learning_rate": 6.242624785351235e-07,
+ "loss": 0.04646766185760498,
+ "num_tokens": 4861629.0,
+ "reward": 0.3379192352294922,
+ "reward_std": 0.3291850686073303,
+ "rewards/true_env_reward_fn/mean": 0.3379192352294922,
+ "rewards/true_env_reward_fn/std": 0.3291850984096527,
+ "step": 197,
+ "step_time": 9.615110594999805
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 82.20833587646484,
+ "completions/mean_terminated_length": 82.20833587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0347496271133423,
+ "epoch": 4.829268292682927,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.038518842309713364,
+ "kl": 1.8024265045823995e-05,
+ "learning_rate": 6.201346823477302e-07,
+ "loss": 0.005069371312856674,
+ "num_tokens": 4880671.0,
+ "reward": 0.5492597818374634,
+ "reward_std": 0.20638948678970337,
+ "rewards/true_env_reward_fn/mean": 0.5492597222328186,
+ "rewards/true_env_reward_fn/std": 0.20638947188854218,
+ "step": 198,
+ "step_time": 10.832162847999825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 65.60417175292969,
+ "completions/mean_terminated_length": 65.60417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2905827760696411,
+ "epoch": 4.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0785721018910408,
+ "kl": 7.324252146645449e-05,
+ "learning_rate": 6.159981782731473e-07,
+ "loss": -0.021710166707634926,
+ "num_tokens": 4904516.0,
+ "reward": 0.4835298955440521,
+ "reward_std": 0.27475905418395996,
+ "rewards/true_env_reward_fn/mean": 0.4835298955440521,
+ "rewards/true_env_reward_fn/std": 0.27475905418395996,
+ "step": 199,
+ "step_time": 10.94759418800004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.20833587646484,
+ "completions/mean_terminated_length": 70.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.284969449043274,
+ "epoch": 4.878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0855984017252922,
+ "kl": 4.8397108912467957e-05,
+ "learning_rate": 6.118532661432811e-07,
+ "loss": 0.03759150952100754,
+ "num_tokens": 4928134.0,
+ "reward": 0.5413213968276978,
+ "reward_std": 0.18470171093940735,
+ "rewards/true_env_reward_fn/mean": 0.5413213968276978,
+ "rewards/true_env_reward_fn/std": 0.18470169603824615,
+ "step": 200,
+ "step_time": 10.322844021000492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 62.083335876464844,
+ "completions/mean_terminated_length": 62.083335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.319727510213852,
+ "epoch": 4.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08774503320455551,
+ "kl": 6.86226876496221e-05,
+ "learning_rate": 6.077002463994907e-07,
+ "loss": 0.058894164860248566,
+ "num_tokens": 4956466.0,
+ "reward": 0.41384777426719666,
+ "reward_std": 0.17193447053432465,
+ "rewards/true_env_reward_fn/mean": 0.41384777426719666,
+ "rewards/true_env_reward_fn/std": 0.17193445563316345,
+ "step": 201,
+ "step_time": 10.146928047999609
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 59.395835876464844,
+ "completions/mean_terminated_length": 59.395835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1935442388057709,
+ "epoch": 4.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08328510820865631,
+ "kl": 7.948942857183283e-05,
+ "learning_rate": 6.035394200708103e-07,
+ "loss": 0.045964501798152924,
+ "num_tokens": 4978733.0,
+ "reward": 0.4896667003631592,
+ "reward_std": 0.3830767571926117,
+ "rewards/true_env_reward_fn/mean": 0.4896667003631592,
+ "rewards/true_env_reward_fn/std": 0.3830767571926117,
+ "step": 202,
+ "step_time": 10.025533761000133
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 62.22916793823242,
+ "completions/mean_terminated_length": 62.22916793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2698631286621094,
+ "epoch": 4.951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07158155739307404,
+ "kl": 8.286665615742095e-05,
+ "learning_rate": 5.993710887521302e-07,
+ "loss": -0.03551984950900078,
+ "num_tokens": 5009440.0,
+ "reward": 0.45639634132385254,
+ "reward_std": 0.1837630569934845,
+ "rewards/true_env_reward_fn/mean": 0.45639634132385254,
+ "rewards/true_env_reward_fn/std": 0.1837630420923233,
+ "step": 203,
+ "step_time": 9.823523802999716
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2825455367565155,
+ "epoch": 4.975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07431793212890625,
+ "kl": 7.247529447340639e-05,
+ "learning_rate": 5.951955545823342e-07,
+ "loss": -0.02418600022792816,
+ "num_tokens": 5036036.0,
+ "reward": 0.4476773142814636,
+ "reward_std": 0.20447416603565216,
+ "rewards/true_env_reward_fn/mean": 0.44767728447914124,
+ "rewards/true_env_reward_fn/std": 0.20447418093681335,
+ "step": 204,
+ "step_time": 11.960790695000014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2438389956951141,
+ "epoch": 5.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06833480298519135,
+ "kl": 5.646793260893901e-05,
+ "learning_rate": 5.91013120222401e-07,
+ "loss": 0.0021926667541265488,
+ "num_tokens": 5063228.0,
+ "reward": 0.4300723075866699,
+ "reward_std": 0.12116922438144684,
+ "rewards/true_env_reward_fn/mean": 0.4300723075866699,
+ "rewards/true_env_reward_fn/std": 0.12116922438144684,
+ "step": 205,
+ "step_time": 9.874485716000436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 205.0,
+ "completions/max_terminated_length": 205.0,
+ "completions/mean_length": 66.1875,
+ "completions/mean_terminated_length": 66.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2645181119441986,
+ "epoch": 5.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08248262107372284,
+ "kl": 9.333990783488844e-05,
+ "learning_rate": 5.868240888334652e-07,
+ "loss": 0.12253005057573318,
+ "num_tokens": 5093373.0,
+ "reward": 0.28542038798332214,
+ "reward_std": 0.326623797416687,
+ "rewards/true_env_reward_fn/mean": 0.28542038798332214,
+ "rewards/true_env_reward_fn/std": 0.326623797416687,
+ "step": 206,
+ "step_time": 15.706792760000553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.16667175292969,
+ "completions/mean_terminated_length": 67.16667175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.34083291888237,
+ "epoch": 5.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07931552827358246,
+ "kl": 0.00010715152166085318,
+ "learning_rate": 5.826287640548424e-07,
+ "loss": 0.0005592256784439087,
+ "num_tokens": 5118933.0,
+ "reward": 0.5035215616226196,
+ "reward_std": 0.304157018661499,
+ "rewards/true_env_reward_fn/mean": 0.5035215020179749,
+ "rewards/true_env_reward_fn/std": 0.30415698885917664,
+ "step": 207,
+ "step_time": 10.04168460900064
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 73.4375,
+ "completions/mean_terminated_length": 73.4375,
+ "completions/min_length": 11.0,
+ "completions/min_terminated_length": 11.0,
+ "entropy": 1.1821868121623993,
+ "epoch": 5.073170731707317,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07619761675596237,
+ "kl": 0.00013189401397539768,
+ "learning_rate": 5.784274499820213e-07,
+ "loss": -0.011967688798904419,
+ "num_tokens": 5148682.0,
+ "reward": 0.36097532510757446,
+ "reward_std": 0.29208436608314514,
+ "rewards/true_env_reward_fn/mean": 0.3609752953052521,
+ "rewards/true_env_reward_fn/std": 0.29208436608314514,
+ "step": 208,
+ "step_time": 15.226898961000188
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 256.0,
+ "completions/max_terminated_length": 256.0,
+ "completions/mean_length": 75.39583587646484,
+ "completions/mean_terminated_length": 75.39583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2636725008487701,
+ "epoch": 5.097560975609756,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06586338579654694,
+ "kl": 1.9426477138040354e-05,
+ "learning_rate": 5.742204511446203e-07,
+ "loss": -0.047095417976379395,
+ "num_tokens": 5173205.0,
+ "reward": 0.5261925458908081,
+ "reward_std": 0.2689943313598633,
+ "rewards/true_env_reward_fn/mean": 0.5261925458908081,
+ "rewards/true_env_reward_fn/std": 0.2689943313598633,
+ "step": 209,
+ "step_time": 23.690397457000017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 54.3125,
+ "completions/mean_terminated_length": 54.3125,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2415392696857452,
+ "epoch": 5.121951219512195,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0990133062005043,
+ "kl": 7.119746442185715e-05,
+ "learning_rate": 5.700080724843146e-07,
+ "loss": -0.030588299036026,
+ "num_tokens": 5199892.0,
+ "reward": 0.3418487012386322,
+ "reward_std": 0.36353805661201477,
+ "rewards/true_env_reward_fn/mean": 0.3418487012386322,
+ "rewards/true_env_reward_fn/std": 0.3635380268096924,
+ "step": 210,
+ "step_time": 9.864614251999683
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2508135735988617,
+ "epoch": 5.146341463414634,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10835102945566177,
+ "kl": 0.00012395972225931473,
+ "learning_rate": 5.657906193327324e-07,
+ "loss": 0.08621911704540253,
+ "num_tokens": 5220141.0,
+ "reward": 0.4987506866455078,
+ "reward_std": 0.27843451499938965,
+ "rewards/true_env_reward_fn/mean": 0.4987506866455078,
+ "rewards/true_env_reward_fn/std": 0.27843451499938965,
+ "step": 211,
+ "step_time": 9.983622502000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1611086130142212,
+ "epoch": 5.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08423946797847748,
+ "kl": 4.4347343191475375e-05,
+ "learning_rate": 5.615683973893234e-07,
+ "loss": 0.08670675754547119,
+ "num_tokens": 5244470.0,
+ "reward": 0.4252437949180603,
+ "reward_std": 0.2996494472026825,
+ "rewards/true_env_reward_fn/mean": 0.4252437651157379,
+ "rewards/true_env_reward_fn/std": 0.2996494472026825,
+ "step": 212,
+ "step_time": 18.68646409699977
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3110275566577911,
+ "epoch": 5.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06817487627267838,
+ "kl": 0.00010393545744591393,
+ "learning_rate": 5.573417126992002e-07,
+ "loss": 0.04062439873814583,
+ "num_tokens": 5271369.0,
+ "reward": 0.4340522885322571,
+ "reward_std": 0.26940545439720154,
+ "rewards/true_env_reward_fn/mean": 0.4340522587299347,
+ "rewards/true_env_reward_fn/std": 0.26940542459487915,
+ "step": 213,
+ "step_time": 11.3590317649996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 54.4375,
+ "completions/mean_terminated_length": 54.4375,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.2018881738185883,
+ "epoch": 5.219512195121951,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10059589147567749,
+ "kl": 0.00011373830420779996,
+ "learning_rate": 5.531108716309547e-07,
+ "loss": -0.09816272556781769,
+ "num_tokens": 5288638.0,
+ "reward": 0.5691710710525513,
+ "reward_std": 0.25253745913505554,
+ "rewards/true_env_reward_fn/mean": 0.5691710710525513,
+ "rewards/true_env_reward_fn/std": 0.25253745913505554,
+ "step": 214,
+ "step_time": 9.633293675000004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 67.22917175292969,
+ "completions/mean_terminated_length": 67.22917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1958912312984467,
+ "epoch": 5.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0779944583773613,
+ "kl": 7.025236300250981e-05,
+ "learning_rate": 5.488761808544509e-07,
+ "loss": 0.03371567651629448,
+ "num_tokens": 5312081.0,
+ "reward": 0.44519662857055664,
+ "reward_std": 0.22201679646968842,
+ "rewards/true_env_reward_fn/mean": 0.44519662857055664,
+ "rewards/true_env_reward_fn/std": 0.22201678156852722,
+ "step": 215,
+ "step_time": 12.463993090999338
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 68.27083587646484,
+ "completions/mean_terminated_length": 68.27083587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.2146256864070892,
+ "epoch": 5.2682926829268295,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06834157556295395,
+ "kl": 0.00010125362859980669,
+ "learning_rate": 5.446379473185971e-07,
+ "loss": -0.02198379673063755,
+ "num_tokens": 5334238.0,
+ "reward": 0.5273472666740417,
+ "reward_std": 0.21954773366451263,
+ "rewards/true_env_reward_fn/mean": 0.5273472666740417,
+ "rewards/true_env_reward_fn/std": 0.21954771876335144,
+ "step": 216,
+ "step_time": 15.126136884000061
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 61.333335876464844,
+ "completions/mean_terminated_length": 61.333335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2872387170791626,
+ "epoch": 5.2926829268292686,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09578828513622284,
+ "kl": 7.404103780572768e-05,
+ "learning_rate": 5.403964782290961e-07,
+ "loss": -0.041091397404670715,
+ "num_tokens": 5351798.0,
+ "reward": 0.5464547872543335,
+ "reward_std": 0.16224616765975952,
+ "rewards/true_env_reward_fn/mean": 0.5464547276496887,
+ "rewards/true_env_reward_fn/std": 0.16224615275859833,
+ "step": 217,
+ "step_time": 7.2901647220001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 84.89583587646484,
+ "completions/mean_terminated_length": 84.89583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.4322342276573181,
+ "epoch": 5.317073170731708,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0667513981461525,
+ "kl": 2.497344303264981e-05,
+ "learning_rate": 5.361520810261778e-07,
+ "loss": -0.06392769515514374,
+ "num_tokens": 5375369.0,
+ "reward": 0.5213420391082764,
+ "reward_std": 0.1436246931552887,
+ "rewards/true_env_reward_fn/mean": 0.5213419795036316,
+ "rewards/true_env_reward_fn/std": 0.1436246782541275,
+ "step": 218,
+ "step_time": 12.217936152999755
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 72.39583587646484,
+ "completions/mean_terminated_length": 72.39583587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2382279634475708,
+ "epoch": 5.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07372384518384933,
+ "kl": 9.740726909512887e-05,
+ "learning_rate": 5.319050633623141e-07,
+ "loss": 0.010912742465734482,
+ "num_tokens": 5397956.0,
+ "reward": 0.5623860359191895,
+ "reward_std": 0.285375714302063,
+ "rewards/true_env_reward_fn/mean": 0.5623860359191895,
+ "rewards/true_env_reward_fn/std": 0.285375714302063,
+ "step": 219,
+ "step_time": 13.449634822999997
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 65.39583587646484,
+ "completions/mean_terminated_length": 65.39583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2669419348239899,
+ "epoch": 5.365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07886125147342682,
+ "kl": 0.00011165817522851285,
+ "learning_rate": 5.276557330799203e-07,
+ "loss": -0.02433733269572258,
+ "num_tokens": 5418511.0,
+ "reward": 0.6075118780136108,
+ "reward_std": 0.30748677253723145,
+ "rewards/true_env_reward_fn/mean": 0.6075118780136108,
+ "rewards/true_env_reward_fn/std": 0.30748677253723145,
+ "step": 220,
+ "step_time": 12.051496982000117
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2560602128505707,
+ "epoch": 5.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08607357740402222,
+ "kl": 8.424731549894204e-05,
+ "learning_rate": 5.234043981890393e-07,
+ "loss": 0.013634156435728073,
+ "num_tokens": 5439690.0,
+ "reward": 0.5080039501190186,
+ "reward_std": 0.21975299715995789,
+ "rewards/true_env_reward_fn/mean": 0.5080038905143738,
+ "rewards/true_env_reward_fn/std": 0.21975299715995789,
+ "step": 221,
+ "step_time": 10.435893627000496
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3081265985965729,
+ "epoch": 5.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0876006931066513,
+ "kl": 9.127605972025776e-05,
+ "learning_rate": 5.191513668450177e-07,
+ "loss": -0.01968565583229065,
+ "num_tokens": 5468216.0,
+ "reward": 0.3679848611354828,
+ "reward_std": 0.237859845161438,
+ "rewards/true_env_reward_fn/mean": 0.3679848611354828,
+ "rewards/true_env_reward_fn/std": 0.237859845161438,
+ "step": 222,
+ "step_time": 11.524256381999294
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 67.66667175292969,
+ "completions/mean_terminated_length": 67.66667175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2990808486938477,
+ "epoch": 5.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07685395330190659,
+ "kl": 0.00014209141045284923,
+ "learning_rate": 5.148969473261679e-07,
+ "loss": -0.01107364147901535,
+ "num_tokens": 5488040.0,
+ "reward": 0.5435788035392761,
+ "reward_std": 0.35419100522994995,
+ "rewards/true_env_reward_fn/mean": 0.5435788035392761,
+ "rewards/true_env_reward_fn/std": 0.35419100522994995,
+ "step": 223,
+ "step_time": 11.945272217000365
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.284348964691162,
+ "epoch": 5.463414634146342,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.058678388595581055,
+ "kl": 3.3754420201148605e-05,
+ "learning_rate": 5.106414480114238e-07,
+ "loss": -0.01714605651795864,
+ "num_tokens": 5513005.0,
+ "reward": 0.43836766481399536,
+ "reward_std": 0.26744046807289124,
+ "rewards/true_env_reward_fn/mean": 0.438367635011673,
+ "rewards/true_env_reward_fn/std": 0.26744046807289124,
+ "step": 224,
+ "step_time": 10.563708176000091
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3357127904891968,
+ "epoch": 5.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07269197702407837,
+ "kl": 3.727909597728285e-05,
+ "learning_rate": 5.063851773579869e-07,
+ "loss": -0.01076439768075943,
+ "num_tokens": 5530655.0,
+ "reward": 0.6185358762741089,
+ "reward_std": 0.19721543788909912,
+ "rewards/true_env_reward_fn/mean": 0.6185358762741089,
+ "rewards/true_env_reward_fn/std": 0.19721543788909912,
+ "step": 225,
+ "step_time": 10.372150705000422
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 64.35417175292969,
+ "completions/mean_terminated_length": 64.35417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2975924611091614,
+ "epoch": 5.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0875314474105835,
+ "kl": 9.186910938296933e-05,
+ "learning_rate": 5.021284438789693e-07,
+ "loss": 0.048179637640714645,
+ "num_tokens": 5554032.0,
+ "reward": 0.45552024245262146,
+ "reward_std": 0.31553781032562256,
+ "rewards/true_env_reward_fn/mean": 0.45552024245262146,
+ "rewards/true_env_reward_fn/std": 0.31553778052330017,
+ "step": 226,
+ "step_time": 12.18860100899974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 63.208335876464844,
+ "completions/mean_terminated_length": 63.208335876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2268341779708862,
+ "epoch": 5.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08318481594324112,
+ "kl": 8.052505017985823e-05,
+ "learning_rate": 4.978715561210307e-07,
+ "loss": 0.027963606640696526,
+ "num_tokens": 5582898.0,
+ "reward": 0.36310288310050964,
+ "reward_std": 0.3131144344806671,
+ "rewards/true_env_reward_fn/mean": 0.36310288310050964,
+ "rewards/true_env_reward_fn/std": 0.3131144344806671,
+ "step": 227,
+ "step_time": 12.510411257000214
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 64.9375,
+ "completions/mean_terminated_length": 64.9375,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1668500006198883,
+ "epoch": 5.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0727877989411354,
+ "kl": 7.824771091691218e-05,
+ "learning_rate": 4.936148226420132e-07,
+ "loss": -0.01629078947007656,
+ "num_tokens": 5604791.0,
+ "reward": 0.5653349161148071,
+ "reward_std": 0.2849632799625397,
+ "rewards/true_env_reward_fn/mean": 0.5653349161148071,
+ "rewards/true_env_reward_fn/std": 0.2849632799625397,
+ "step": 228,
+ "step_time": 10.083805716999905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 133.0,
+ "completions/max_terminated_length": 133.0,
+ "completions/mean_length": 68.29167175292969,
+ "completions/mean_terminated_length": 68.29167175292969,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.307439923286438,
+ "epoch": 5.585365853658536,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06797961890697479,
+ "kl": 6.601300174224889e-05,
+ "learning_rate": 4.893585519885763e-07,
+ "loss": -0.036383360624313354,
+ "num_tokens": 5631629.0,
+ "reward": 0.49251794815063477,
+ "reward_std": 0.22127023339271545,
+ "rewards/true_env_reward_fn/mean": 0.49251794815063477,
+ "rewards/true_env_reward_fn/std": 0.22127023339271545,
+ "step": 229,
+ "step_time": 13.650024606999523
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 63.395835876464844,
+ "completions/mean_terminated_length": 63.395835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3524385392665863,
+ "epoch": 5.609756097560975,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.12484809756278992,
+ "kl": 0.0002558713749749586,
+ "learning_rate": 4.85103052673832e-07,
+ "loss": -0.0045075975358486176,
+ "num_tokens": 5666712.0,
+ "reward": 0.2605232000350952,
+ "reward_std": 0.22926004230976105,
+ "rewards/true_env_reward_fn/mean": 0.2605231702327728,
+ "rewards/true_env_reward_fn/std": 0.22926005721092224,
+ "step": 230,
+ "step_time": 15.322059910999542
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2202666401863098,
+ "epoch": 5.634146341463414,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07976125925779343,
+ "kl": 8.684267686476232e-05,
+ "learning_rate": 4.808486331549823e-07,
+ "loss": 0.012383833527565002,
+ "num_tokens": 5702004.0,
+ "reward": 0.20571085810661316,
+ "reward_std": 0.2265808880329132,
+ "rewards/true_env_reward_fn/mean": 0.20571084320545197,
+ "rewards/true_env_reward_fn/std": 0.2265808880329132,
+ "step": 231,
+ "step_time": 14.180213787999492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 312.0,
+ "completions/max_terminated_length": 312.0,
+ "completions/mean_length": 78.83333587646484,
+ "completions/mean_terminated_length": 78.83333587646484,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3096380531787872,
+ "epoch": 5.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08384906500577927,
+ "kl": 8.733692084206268e-05,
+ "learning_rate": 4.7659560181096067e-07,
+ "loss": 0.02098608762025833,
+ "num_tokens": 5729780.0,
+ "reward": 0.4599723219871521,
+ "reward_std": 0.21680118143558502,
+ "rewards/true_env_reward_fn/mean": 0.4599722921848297,
+ "rewards/true_env_reward_fn/std": 0.21680118143558502,
+ "step": 232,
+ "step_time": 21.68401394800003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 53.520835876464844,
+ "completions/mean_terminated_length": 53.520835876464844,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2158988416194916,
+ "epoch": 5.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08225176483392715,
+ "kl": 9.274652438762132e-05,
+ "learning_rate": 4.7234426692007977e-07,
+ "loss": -0.034079790115356445,
+ "num_tokens": 5753397.0,
+ "reward": 0.5273070335388184,
+ "reward_std": 0.2194610834121704,
+ "rewards/true_env_reward_fn/mean": 0.5273070335388184,
+ "rewards/true_env_reward_fn/std": 0.21946106851100922,
+ "step": 233,
+ "step_time": 7.915307780000603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 78.0,
+ "completions/mean_terminated_length": 78.0,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2362721860408783,
+ "epoch": 5.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06290840357542038,
+ "kl": 7.484563047910342e-05,
+ "learning_rate": 4.6809493663768575e-07,
+ "loss": -0.008873865008354187,
+ "num_tokens": 5778869.0,
+ "reward": 0.5283026695251465,
+ "reward_std": 0.195759579539299,
+ "rewards/true_env_reward_fn/mean": 0.5283026695251465,
+ "rewards/true_env_reward_fn/std": 0.195759579539299,
+ "step": 234,
+ "step_time": 10.278297286999987
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 64.47917175292969,
+ "completions/mean_terminated_length": 64.47917175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3364675641059875,
+ "epoch": 5.7317073170731705,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09685558825731277,
+ "kl": 0.00013002969353692606,
+ "learning_rate": 4.638479189738224e-07,
+ "loss": 0.05070740357041359,
+ "num_tokens": 5804436.0,
+ "reward": 0.44921523332595825,
+ "reward_std": 0.22693434357643127,
+ "rewards/true_env_reward_fn/mean": 0.44921520352363586,
+ "rewards/true_env_reward_fn/std": 0.22693434357643127,
+ "step": 235,
+ "step_time": 10.754199091999908
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2267529368400574,
+ "epoch": 5.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.10451620817184448,
+ "kl": 6.754635069228243e-05,
+ "learning_rate": 4.596035217709039e-07,
+ "loss": -0.02925686165690422,
+ "num_tokens": 5826895.0,
+ "reward": 0.49523353576660156,
+ "reward_std": 0.15093794465065002,
+ "rewards/true_env_reward_fn/mean": 0.49523353576660156,
+ "rewards/true_env_reward_fn/std": 0.15093792974948883,
+ "step": 236,
+ "step_time": 9.666070583999954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 67.79167175292969,
+ "completions/mean_terminated_length": 67.79167175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2110519707202911,
+ "epoch": 5.780487804878049,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07890205830335617,
+ "kl": 0.0001245876064785989,
+ "learning_rate": 4.5536205268140286e-07,
+ "loss": -0.03693925589323044,
+ "num_tokens": 5859973.0,
+ "reward": 0.2979053258895874,
+ "reward_std": 0.23015481233596802,
+ "rewards/true_env_reward_fn/mean": 0.297905296087265,
+ "rewards/true_env_reward_fn/std": 0.2301548272371292,
+ "step": 237,
+ "step_time": 13.971699990999241
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 65.52083587646484,
+ "completions/mean_terminated_length": 65.52083587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2291057407855988,
+ "epoch": 5.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09152546525001526,
+ "kl": 0.00013101351214572787,
+ "learning_rate": 4.511238191455491e-07,
+ "loss": 0.043641336262226105,
+ "num_tokens": 5886118.0,
+ "reward": 0.41334670782089233,
+ "reward_std": 0.19480590522289276,
+ "rewards/true_env_reward_fn/mean": 0.41334667801856995,
+ "rewards/true_env_reward_fn/std": 0.19480590522289276,
+ "step": 238,
+ "step_time": 10.805698846999803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 58.833335876464844,
+ "completions/mean_terminated_length": 58.833335876464844,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3134913444519043,
+ "epoch": 5.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07343924045562744,
+ "kl": 0.00019655993492051493,
+ "learning_rate": 4.4688912836904533e-07,
+ "loss": 0.015309082344174385,
+ "num_tokens": 5908334.0,
+ "reward": 0.4438478946685791,
+ "reward_std": 0.27188754081726074,
+ "rewards/true_env_reward_fn/mean": 0.4438478946685791,
+ "rewards/true_env_reward_fn/std": 0.27188754081726074,
+ "step": 239,
+ "step_time": 9.688736522
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 464.0,
+ "completions/max_terminated_length": 464.0,
+ "completions/mean_length": 79.14583587646484,
+ "completions/mean_terminated_length": 79.14583587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3148745000362396,
+ "epoch": 5.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06567981839179993,
+ "kl": 4.445325248525478e-05,
+ "learning_rate": 4.4265828730079977e-07,
+ "loss": 0.021218176931142807,
+ "num_tokens": 5929645.0,
+ "reward": 0.5866342186927795,
+ "reward_std": 0.13780196011066437,
+ "rewards/true_env_reward_fn/mean": 0.5866342186927795,
+ "rewards/true_env_reward_fn/std": 0.13780196011066437,
+ "step": 240,
+ "step_time": 27.046819901000163
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 364.0,
+ "completions/max_terminated_length": 364.0,
+ "completions/mean_length": 70.33333587646484,
+ "completions/mean_terminated_length": 70.33333587646484,
+ "completions/min_length": 15.0,
+ "completions/min_terminated_length": 15.0,
+ "entropy": 1.2104995846748352,
+ "epoch": 5.878048780487805,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.042626503854990005,
+ "kl": 6.413207393052289e-05,
+ "learning_rate": 4.3843160261067653e-07,
+ "loss": -0.0065308245830237865,
+ "num_tokens": 5950005.0,
+ "reward": 0.5593677759170532,
+ "reward_std": 0.23993276059627533,
+ "rewards/true_env_reward_fn/mean": 0.5593677163124084,
+ "rewards/true_env_reward_fn/std": 0.23993274569511414,
+ "step": 241,
+ "step_time": 22.238758486000734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2132116258144379,
+ "epoch": 5.902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05718924477696419,
+ "kl": 6.140609366411809e-05,
+ "learning_rate": 4.342093806672678e-07,
+ "loss": 0.012734346091747284,
+ "num_tokens": 5974626.0,
+ "reward": 0.4349059462547302,
+ "reward_std": 0.24915602803230286,
+ "rewards/true_env_reward_fn/mean": 0.43490591645240784,
+ "rewards/true_env_reward_fn/std": 0.24915601313114166,
+ "step": 242,
+ "step_time": 9.239750460000323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 67.04167175292969,
+ "completions/mean_terminated_length": 57.574466705322266,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.7212344706058502,
+ "epoch": 5.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08872146159410477,
+ "kl": 0.00011613740025495645,
+ "learning_rate": 4.2999192751568557e-07,
+ "loss": -0.021363887935876846,
+ "num_tokens": 5997300.0,
+ "reward": 0.4436037540435791,
+ "reward_std": 0.28323379158973694,
+ "rewards/true_env_reward_fn/mean": 0.4436037540435791,
+ "rewards/true_env_reward_fn/std": 0.28323376178741455,
+ "step": 243,
+ "step_time": 29.767976787999487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 61.083335876464844,
+ "completions/mean_terminated_length": 61.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2353634238243103,
+ "epoch": 5.951219512195122,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07658001780509949,
+ "kl": 9.086773115996039e-05,
+ "learning_rate": 4.257795488553798e-07,
+ "loss": -0.005057391710579395,
+ "num_tokens": 6021752.0,
+ "reward": 0.49971556663513184,
+ "reward_std": 0.2643933594226837,
+ "rewards/true_env_reward_fn/mean": 0.49971556663513184,
+ "rewards/true_env_reward_fn/std": 0.2643933594226837,
+ "step": 244,
+ "step_time": 9.764708648999658
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 86.72917175292969,
+ "completions/mean_terminated_length": 77.68084716796875,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2082330882549286,
+ "epoch": 5.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.059102799743413925,
+ "kl": 5.0710960749711376e-05,
+ "learning_rate": 4.215725500179787e-07,
+ "loss": 0.17436102032661438,
+ "num_tokens": 6049747.0,
+ "reward": 0.46306928992271423,
+ "reward_std": 0.26072925329208374,
+ "rewards/true_env_reward_fn/mean": 0.46306928992271423,
+ "rewards/true_env_reward_fn/std": 0.26072925329208374,
+ "step": 245,
+ "step_time": 31.997988874000384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 61.35416793823242,
+ "completions/mean_terminated_length": 61.35416793823242,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2454268038272858,
+ "epoch": 6.0,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07332625985145569,
+ "kl": 0.0001107546740968246,
+ "learning_rate": 4.1737123594515755e-07,
+ "loss": -0.013426866382360458,
+ "num_tokens": 6072668.0,
+ "reward": 0.45415109395980835,
+ "reward_std": 0.2937946617603302,
+ "rewards/true_env_reward_fn/mean": 0.45415106415748596,
+ "rewards/true_env_reward_fn/std": 0.2937946617603302,
+ "step": 246,
+ "step_time": 9.587768273000165
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 71.5625,
+ "completions/mean_terminated_length": 71.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1738699078559875,
+ "epoch": 6.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06093747541308403,
+ "kl": 7.893411338955048e-05,
+ "learning_rate": 4.131759111665348e-07,
+ "loss": 0.022837676107883453,
+ "num_tokens": 6095263.0,
+ "reward": 0.5188159346580505,
+ "reward_std": 0.3265886902809143,
+ "rewards/true_env_reward_fn/mean": 0.5188159346580505,
+ "rewards/true_env_reward_fn/std": 0.3265886902809143,
+ "step": 247,
+ "step_time": 14.232978527999876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3300949931144714,
+ "epoch": 6.048780487804878,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08098509907722473,
+ "kl": 0.00010828049380506855,
+ "learning_rate": 4.0898687977759887e-07,
+ "loss": 0.01631344109773636,
+ "num_tokens": 6125380.0,
+ "reward": 0.3338983952999115,
+ "reward_std": 0.19050820171833038,
+ "rewards/true_env_reward_fn/mean": 0.3338983952999115,
+ "rewards/true_env_reward_fn/std": 0.19050820171833038,
+ "step": 248,
+ "step_time": 13.248441182000079
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 396.0,
+ "completions/max_terminated_length": 396.0,
+ "completions/mean_length": 88.14583587646484,
+ "completions/mean_terminated_length": 88.14583587646484,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3161276876926422,
+ "epoch": 6.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07784765958786011,
+ "kl": 4.819030800717883e-05,
+ "learning_rate": 4.0480444541766575e-07,
+ "loss": 0.12299837917089462,
+ "num_tokens": 6151955.0,
+ "reward": 0.5446512699127197,
+ "reward_std": 0.2611033618450165,
+ "rewards/true_env_reward_fn/mean": 0.544651210308075,
+ "rewards/true_env_reward_fn/std": 0.2611033618450165,
+ "step": 249,
+ "step_time": 25.232192139999825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 69.97917175292969,
+ "completions/mean_terminated_length": 69.97917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2318329215049744,
+ "epoch": 6.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06498592346906662,
+ "kl": 3.854301621686318e-05,
+ "learning_rate": 4.0062891124787e-07,
+ "loss": -0.04302535578608513,
+ "num_tokens": 6174898.0,
+ "reward": 0.6081289052963257,
+ "reward_std": 0.24437586963176727,
+ "rewards/true_env_reward_fn/mean": 0.6081289052963257,
+ "rewards/true_env_reward_fn/std": 0.24437588453292847,
+ "step": 250,
+ "step_time": 10.385816780999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 60.91666793823242,
+ "completions/mean_terminated_length": 60.91666793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2425517737865448,
+ "epoch": 6.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08402577042579651,
+ "kl": 0.00010545800250838511,
+ "learning_rate": 3.9646057992918966e-07,
+ "loss": 0.006328321993350983,
+ "num_tokens": 6203582.0,
+ "reward": 0.30250340700149536,
+ "reward_std": 0.3084425926208496,
+ "rewards/true_env_reward_fn/mean": 0.30250340700149536,
+ "rewards/true_env_reward_fn/std": 0.3084425628185272,
+ "step": 251,
+ "step_time": 9.779451584999606
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.5,
+ "completions/mean_terminated_length": 65.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2977190017700195,
+ "epoch": 6.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372989296913147,
+ "kl": 0.00017708011364447884,
+ "learning_rate": 3.9229975360050934e-07,
+ "loss": 0.04882372170686722,
+ "num_tokens": 6227486.0,
+ "reward": 0.5265982151031494,
+ "reward_std": 0.31736499071121216,
+ "rewards/true_env_reward_fn/mean": 0.5265981554985046,
+ "rewards/true_env_reward_fn/std": 0.31736496090888977,
+ "step": 252,
+ "step_time": 10.318136508000407
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 71.125,
+ "completions/mean_terminated_length": 71.125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3135383129119873,
+ "epoch": 6.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05451377481222153,
+ "kl": 6.562464477610774e-05,
+ "learning_rate": 3.8814673385671893e-07,
+ "loss": 0.01392640545964241,
+ "num_tokens": 6255436.0,
+ "reward": 0.4130678176879883,
+ "reward_std": 0.1573377549648285,
+ "rewards/true_env_reward_fn/mean": 0.4130678176879883,
+ "rewards/true_env_reward_fn/std": 0.15733776986598969,
+ "step": 253,
+ "step_time": 9.396596211999622
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 73.85417175292969,
+ "completions/mean_terminated_length": 73.85417175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3749704957008362,
+ "epoch": 6.195121951219512,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05664386227726936,
+ "kl": 6.744195252395002e-05,
+ "learning_rate": 3.840018217268526e-07,
+ "loss": -0.04205852374434471,
+ "num_tokens": 6281557.0,
+ "reward": 0.5188004970550537,
+ "reward_std": 0.2932124733924866,
+ "rewards/true_env_reward_fn/mean": 0.5188004970550537,
+ "rewards/true_env_reward_fn/std": 0.2932124435901642,
+ "step": 254,
+ "step_time": 19.77463799499992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2866049408912659,
+ "epoch": 6.219512195121951,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06639927625656128,
+ "kl": 5.235667686065426e-05,
+ "learning_rate": 3.798653176522696e-07,
+ "loss": -0.019558893516659737,
+ "num_tokens": 6302850.0,
+ "reward": 0.5025100111961365,
+ "reward_std": 0.1717289537191391,
+ "rewards/true_env_reward_fn/mean": 0.5025100111961365,
+ "rewards/true_env_reward_fn/std": 0.1717289537191391,
+ "step": 255,
+ "step_time": 9.634558264001043
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.3105897009372711,
+ "epoch": 6.2439024390243905,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07659970223903656,
+ "kl": 8.12946273072157e-05,
+ "learning_rate": 3.7573752146487636e-07,
+ "loss": 0.05201243981719017,
+ "num_tokens": 6329269.0,
+ "reward": 0.5027985572814941,
+ "reward_std": 0.2521378695964813,
+ "rewards/true_env_reward_fn/mean": 0.5027985572814941,
+ "rewards/true_env_reward_fn/std": 0.2521378993988037,
+ "step": 256,
+ "step_time": 11.07390475200009
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.3154918253421783,
+ "epoch": 6.2682926829268295,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10612925887107849,
+ "kl": 0.00010100230429088697,
+ "learning_rate": 3.7161873236539386e-07,
+ "loss": 0.0214182510972023,
+ "num_tokens": 6356233.0,
+ "reward": 0.2817384600639343,
+ "reward_std": 0.3363598883152008,
+ "rewards/true_env_reward_fn/mean": 0.2817384600639343,
+ "rewards/true_env_reward_fn/std": 0.3363598883152008,
+ "step": 257,
+ "step_time": 10.784447634000117
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 62.770835876464844,
+ "completions/mean_terminated_length": 62.770835876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1986172497272491,
+ "epoch": 6.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08129199594259262,
+ "kl": 0.00011449725025158841,
+ "learning_rate": 3.6750924890166923e-07,
+ "loss": -0.05941678211092949,
+ "num_tokens": 6375046.0,
+ "reward": 0.5760313272476196,
+ "reward_std": 0.24430812895298004,
+ "rewards/true_env_reward_fn/mean": 0.5760312676429749,
+ "rewards/true_env_reward_fn/std": 0.24430814385414124,
+ "step": 258,
+ "step_time": 9.467202022999572
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2520000636577606,
+ "epoch": 6.317073170731708,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.06876237690448761,
+ "kl": 0.0001509120993432589,
+ "learning_rate": 3.6340936894703713e-07,
+ "loss": 0.015932239592075348,
+ "num_tokens": 6397298.0,
+ "reward": 0.4944794774055481,
+ "reward_std": 0.24476772546768188,
+ "rewards/true_env_reward_fn/mean": 0.4944794476032257,
+ "rewards/true_env_reward_fn/std": 0.24476774036884308,
+ "step": 259,
+ "step_time": 10.47640546699995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 66.9375,
+ "completions/mean_terminated_length": 66.9375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3675504922866821,
+ "epoch": 6.341463414634147,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06423573940992355,
+ "kl": 0.00010545238592385431,
+ "learning_rate": 3.593193896787277e-07,
+ "loss": 0.006066613830626011,
+ "num_tokens": 6423927.0,
+ "reward": 0.4633293151855469,
+ "reward_std": 0.2953638732433319,
+ "rewards/true_env_reward_fn/mean": 0.4633293151855469,
+ "rewards/true_env_reward_fn/std": 0.2953638732433319,
+ "step": 260,
+ "step_time": 11.672983966999709
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 74.5,
+ "completions/mean_terminated_length": 74.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3533118963241577,
+ "epoch": 6.365853658536586,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.07830996066331863,
+ "kl": 8.717951732251095e-05,
+ "learning_rate": 3.552396075563257e-07,
+ "loss": -0.021772567182779312,
+ "num_tokens": 6451487.0,
+ "reward": 0.3543766736984253,
+ "reward_std": 0.33776554465293884,
+ "rewards/true_env_reward_fn/mean": 0.3543766736984253,
+ "rewards/true_env_reward_fn/std": 0.33776557445526123,
+ "step": 261,
+ "step_time": 13.734938852000141
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 71.45833587646484,
+ "completions/mean_terminated_length": 71.45833587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1788119971752167,
+ "epoch": 6.390243902439025,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0994381383061409,
+ "kl": 0.00011037426884286106,
+ "learning_rate": 3.511703183002827e-07,
+ "loss": 0.008012240752577782,
+ "num_tokens": 6476141.0,
+ "reward": 0.45457857847213745,
+ "reward_std": 0.24262367188930511,
+ "rewards/true_env_reward_fn/mean": 0.45457854866981506,
+ "rewards/true_env_reward_fn/std": 0.24262367188930511,
+ "step": 262,
+ "step_time": 10.59573544200066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.293170690536499,
+ "epoch": 6.414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10550480335950851,
+ "kl": 0.00020241059792169835,
+ "learning_rate": 3.4711181687048106e-07,
+ "loss": 0.02680305391550064,
+ "num_tokens": 6499670.0,
+ "reward": 0.33261698484420776,
+ "reward_std": 0.31543657183647156,
+ "rewards/true_env_reward_fn/mean": 0.3326169550418854,
+ "rewards/true_env_reward_fn/std": 0.31543657183647156,
+ "step": 263,
+ "step_time": 10.02452396199942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 348.0,
+ "completions/max_terminated_length": 348.0,
+ "completions/mean_length": 89.83333587646484,
+ "completions/mean_terminated_length": 89.83333587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2982321977615356,
+ "epoch": 6.439024390243903,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06412192434072495,
+ "kl": 6.536830551340245e-05,
+ "learning_rate": 3.4306439744485447e-07,
+ "loss": -0.08396965265274048,
+ "num_tokens": 6522630.0,
+ "reward": 0.5518762469291687,
+ "reward_std": 0.21314994990825653,
+ "rewards/true_env_reward_fn/mean": 0.5518762469291687,
+ "rewards/true_env_reward_fn/std": 0.21314994990825653,
+ "step": 264,
+ "step_time": 20.82168071200067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 63.85416793823242,
+ "completions/mean_terminated_length": 63.85416793823242,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2279660403728485,
+ "epoch": 6.463414634146342,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10119301080703735,
+ "kl": 0.00010847221983567579,
+ "learning_rate": 3.3902835339806456e-07,
+ "loss": 0.01589711755514145,
+ "num_tokens": 6548183.0,
+ "reward": 0.4527897238731384,
+ "reward_std": 0.2534019947052002,
+ "rewards/true_env_reward_fn/mean": 0.45278969407081604,
+ "rewards/true_env_reward_fn/std": 0.2534019649028778,
+ "step": 265,
+ "step_time": 8.63894235699945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 60.54166793823242,
+ "completions/mean_terminated_length": 60.54166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1612891554832458,
+ "epoch": 6.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0825633853673935,
+ "kl": 7.345602170971688e-05,
+ "learning_rate": 3.3500397728023534e-07,
+ "loss": 0.057398565113544464,
+ "num_tokens": 6571537.0,
+ "reward": 0.5196421146392822,
+ "reward_std": 0.18547315895557404,
+ "rewards/true_env_reward_fn/mean": 0.5196421146392822,
+ "rewards/true_env_reward_fn/std": 0.18547315895557404,
+ "step": 266,
+ "step_time": 10.01155260700034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 67.85417175292969,
+ "completions/mean_terminated_length": 67.85417175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2408597469329834,
+ "epoch": 6.512195121951219,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09217255562543869,
+ "kl": 9.46905902310391e-05,
+ "learning_rate": 3.3099156079574867e-07,
+ "loss": 0.026750221848487854,
+ "num_tokens": 6596010.0,
+ "reward": 0.4330406188964844,
+ "reward_std": 0.20423907041549683,
+ "rewards/true_env_reward_fn/mean": 0.4330406188964844,
+ "rewards/true_env_reward_fn/std": 0.20423908531665802,
+ "step": 267,
+ "step_time": 20.002466699999786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 64.83333587646484,
+ "completions/mean_terminated_length": 64.83333587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2907497882843018,
+ "epoch": 6.536585365853659,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07903403043746948,
+ "kl": 0.00015101409735507332,
+ "learning_rate": 3.269913947820998e-07,
+ "loss": 0.02006390690803528,
+ "num_tokens": 6622106.0,
+ "reward": 0.4544076919555664,
+ "reward_std": 0.26717478036880493,
+ "rewards/true_env_reward_fn/mean": 0.4544076919555664,
+ "rewards/true_env_reward_fn/std": 0.2671748101711273,
+ "step": 268,
+ "step_time": 8.818348709999555
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 62.0625,
+ "completions/mean_terminated_length": 62.0625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2150432765483856,
+ "epoch": 6.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09289928525686264,
+ "kl": 9.474463149672374e-05,
+ "learning_rate": 3.230037691888162e-07,
+ "loss": 0.08772514015436172,
+ "num_tokens": 6636101.0,
+ "reward": 0.6059033870697021,
+ "reward_std": 0.23812197148799896,
+ "rewards/true_env_reward_fn/mean": 0.6059033274650574,
+ "rewards/true_env_reward_fn/std": 0.23812197148799896,
+ "step": 269,
+ "step_time": 8.47666211500109
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 5.0,
+ "completions/min_terminated_length": 5.0,
+ "entropy": 1.2745259404182434,
+ "epoch": 6.585365853658536,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09368392825126648,
+ "kl": 0.0001518711451353738,
+ "learning_rate": 3.1902897305644093e-07,
+ "loss": -0.003839995712041855,
+ "num_tokens": 6659877.0,
+ "reward": 0.5029901266098022,
+ "reward_std": 0.3467065095901489,
+ "rewards/true_env_reward_fn/mean": 0.5029900670051575,
+ "rewards/true_env_reward_fn/std": 0.3467065393924713,
+ "step": 270,
+ "step_time": 11.298448464000103
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 67.0625,
+ "completions/mean_terminated_length": 67.0625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.238816499710083,
+ "epoch": 6.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.05819084122776985,
+ "kl": 6.655609695371822e-05,
+ "learning_rate": 3.150672944955818e-07,
+ "loss": -0.02771003544330597,
+ "num_tokens": 6679776.0,
+ "reward": 0.5772091746330261,
+ "reward_std": 0.17815756797790527,
+ "rewards/true_env_reward_fn/mean": 0.5772091746330261,
+ "rewards/true_env_reward_fn/std": 0.17815756797790527,
+ "step": 271,
+ "step_time": 7.902968623000561
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 145.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 75.10417175292969,
+ "completions/mean_terminated_length": 75.10417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3645851016044617,
+ "epoch": 6.634146341463414,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07474905252456665,
+ "kl": 7.497054593841312e-05,
+ "learning_rate": 3.1111902066602724e-07,
+ "loss": 0.07271203398704529,
+ "num_tokens": 6704717.0,
+ "reward": 0.42504438757896423,
+ "reward_std": 0.2500284016132355,
+ "rewards/true_env_reward_fn/mean": 0.42504438757896423,
+ "rewards/true_env_reward_fn/std": 0.2500284016132355,
+ "step": 272,
+ "step_time": 11.259095110999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 67.64583587646484,
+ "completions/mean_terminated_length": 67.64583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2410458326339722,
+ "epoch": 6.658536585365853,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0750170573592186,
+ "kl": 8.587932643422391e-05,
+ "learning_rate": 3.0718443775593225e-07,
+ "loss": -0.019169267266988754,
+ "num_tokens": 6727060.0,
+ "reward": 0.5114523768424988,
+ "reward_std": 0.17165428400039673,
+ "rewards/true_env_reward_fn/mean": 0.5114523768424988,
+ "rewards/true_env_reward_fn/std": 0.17165428400039673,
+ "step": 273,
+ "step_time": 11.097374408000178
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 67.08333587646484,
+ "completions/mean_terminated_length": 67.08333587646484,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.2417791783809662,
+ "epoch": 6.682926829268292,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08969064801931381,
+ "kl": 5.61167857995315e-05,
+ "learning_rate": 3.0326383096107423e-07,
+ "loss": 0.11341358721256256,
+ "num_tokens": 6752824.0,
+ "reward": 0.4772833287715912,
+ "reward_std": 0.31618404388427734,
+ "rewards/true_env_reward_fn/mean": 0.4772833287715912,
+ "rewards/true_env_reward_fn/std": 0.31618407368659973,
+ "step": 274,
+ "step_time": 11.189890726001067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.47917175292969,
+ "completions/mean_terminated_length": 67.47917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.272821843624115,
+ "epoch": 6.7073170731707314,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08874963968992233,
+ "kl": 0.00015055539006425533,
+ "learning_rate": 2.9935748446418065e-07,
+ "loss": 0.02028803899884224,
+ "num_tokens": 6776703.0,
+ "reward": 0.49093031883239746,
+ "reward_std": 0.25163188576698303,
+ "rewards/true_env_reward_fn/mean": 0.49093031883239746,
+ "rewards/true_env_reward_fn/std": 0.25163188576698303,
+ "step": 275,
+ "step_time": 9.941926390000845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 365.0,
+ "completions/max_terminated_length": 365.0,
+ "completions/mean_length": 68.04167175292969,
+ "completions/mean_terminated_length": 68.04167175292969,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.0912510752677917,
+ "epoch": 6.7317073170731705,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.060312703251838684,
+ "kl": 6.840269179519964e-05,
+ "learning_rate": 2.9546568141433e-07,
+ "loss": -0.036469291895627975,
+ "num_tokens": 6799417.0,
+ "reward": 0.5543485879898071,
+ "reward_std": 0.22147472202777863,
+ "rewards/true_env_reward_fn/mean": 0.5543485283851624,
+ "rewards/true_env_reward_fn/std": 0.22147469222545624,
+ "step": 276,
+ "step_time": 22.291117544999906
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.895835876464844,
+ "completions/mean_terminated_length": 59.895835876464844,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.2566059231758118,
+ "epoch": 6.7560975609756095,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09982399642467499,
+ "kl": 0.00020091429905733094,
+ "learning_rate": 2.9158870390642863e-07,
+ "loss": -0.005738064646720886,
+ "num_tokens": 6829804.0,
+ "reward": 0.24754562973976135,
+ "reward_std": 0.23038579523563385,
+ "rewards/true_env_reward_fn/mean": 0.24754561483860016,
+ "rewards/true_env_reward_fn/std": 0.23038578033447266,
+ "step": 277,
+ "step_time": 9.732460060999983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 149.0,
+ "completions/max_terminated_length": 149.0,
+ "completions/mean_length": 71.60417175292969,
+ "completions/mean_terminated_length": 71.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2196767628192902,
+ "epoch": 6.780487804878049,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.03178449347615242,
+ "kl": 5.627466816804372e-05,
+ "learning_rate": 2.8772683296076194e-07,
+ "loss": 0.002586783841252327,
+ "num_tokens": 6852617.0,
+ "reward": 0.5281299352645874,
+ "reward_std": 0.22268518805503845,
+ "rewards/true_env_reward_fn/mean": 0.5281298756599426,
+ "rewards/true_env_reward_fn/std": 0.22268518805503845,
+ "step": 278,
+ "step_time": 11.629210506000163
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 169.0,
+ "completions/max_terminated_length": 169.0,
+ "completions/mean_length": 68.66667175292969,
+ "completions/mean_terminated_length": 68.66667175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1991542279720306,
+ "epoch": 6.804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07567304372787476,
+ "kl": 0.00015181046910583973,
+ "learning_rate": 2.8388034850262646e-07,
+ "loss": -0.04906900227069855,
+ "num_tokens": 6881529.0,
+ "reward": 0.3906375765800476,
+ "reward_std": 0.3576822578907013,
+ "rewards/true_env_reward_fn/mean": 0.3906375467777252,
+ "rewards/true_env_reward_fn/std": 0.3576822876930237,
+ "step": 279,
+ "step_time": 18.253660386000774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 208.0,
+ "completions/max_terminated_length": 208.0,
+ "completions/mean_length": 70.95833587646484,
+ "completions/mean_terminated_length": 70.95833587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.263102412223816,
+ "epoch": 6.829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09104589372873306,
+ "kl": 0.00012209633769089123,
+ "learning_rate": 2.8004952934203837e-07,
+ "loss": 0.06642289459705353,
+ "num_tokens": 6905111.0,
+ "reward": 0.3988339304924011,
+ "reward_std": 0.34396088123321533,
+ "rewards/true_env_reward_fn/mean": 0.39883390069007874,
+ "rewards/true_env_reward_fn/std": 0.3439609110355377,
+ "step": 280,
+ "step_time": 15.519001798998943
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 59.0625,
+ "completions/mean_terminated_length": 59.0625,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.208267629146576,
+ "epoch": 6.853658536585366,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07465670257806778,
+ "kl": 7.14320785846212e-05,
+ "learning_rate": 2.762346531535246e-07,
+ "loss": 0.04124641790986061,
+ "num_tokens": 6926634.0,
+ "reward": 0.5870868563652039,
+ "reward_std": 0.28217118978500366,
+ "rewards/true_env_reward_fn/mean": 0.5870868563652039,
+ "rewards/true_env_reward_fn/std": 0.2821711599826813,
+ "step": 281,
+ "step_time": 11.224198447001072
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 72.02083587646484,
+ "completions/mean_terminated_length": 72.02083587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3142951428890228,
+ "epoch": 6.878048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07364725321531296,
+ "kl": 0.00011016946336894762,
+ "learning_rate": 2.7243599645599574e-07,
+ "loss": 0.004198473412543535,
+ "num_tokens": 6960219.0,
+ "reward": 0.34401482343673706,
+ "reward_std": 0.24699951708316803,
+ "rewards/true_env_reward_fn/mean": 0.3440147936344147,
+ "rewards/true_env_reward_fn/std": 0.24699951708316803,
+ "step": 282,
+ "step_time": 15.210776117999558
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 60.583335876464844,
+ "completions/mean_terminated_length": 60.583335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2013934552669525,
+ "epoch": 6.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07040710747241974,
+ "kl": 7.988750212462037e-05,
+ "learning_rate": 2.686538345927026e-07,
+ "loss": 0.06334929168224335,
+ "num_tokens": 6982879.0,
+ "reward": 0.5139331221580505,
+ "reward_std": 0.19786590337753296,
+ "rewards/true_env_reward_fn/mean": 0.5139331221580505,
+ "rewards/true_env_reward_fn/std": 0.19786591827869415,
+ "step": 283,
+ "step_time": 10.240000448000501
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 60.8125,
+ "completions/mean_terminated_length": 60.8125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2223551571369171,
+ "epoch": 6.926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07971613109111786,
+ "kl": 0.00016717875041649677,
+ "learning_rate": 2.64888441711279e-07,
+ "loss": 0.03706979751586914,
+ "num_tokens": 7003990.0,
+ "reward": 0.5301157832145691,
+ "reward_std": 0.2464885115623474,
+ "rewards/true_env_reward_fn/mean": 0.5301157832145691,
+ "rewards/true_env_reward_fn/std": 0.2464885115623474,
+ "step": 284,
+ "step_time": 9.309556909001913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.309690535068512,
+ "epoch": 6.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09541535377502441,
+ "kl": 0.00014337312313728034,
+ "learning_rate": 2.6114009074386844e-07,
+ "loss": -0.04744558781385422,
+ "num_tokens": 7027733.0,
+ "reward": 0.468932569026947,
+ "reward_std": 0.2396899163722992,
+ "rewards/true_env_reward_fn/mean": 0.46893253922462463,
+ "rewards/true_env_reward_fn/std": 0.2396899312734604,
+ "step": 285,
+ "step_time": 8.91306197799986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1709823310375214,
+ "epoch": 6.975609756097561,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048187412321567535,
+ "kl": 9.718035107653122e-05,
+ "learning_rate": 2.5740905338734306e-07,
+ "loss": 0.010684527456760406,
+ "num_tokens": 7058137.0,
+ "reward": 0.44316989183425903,
+ "reward_std": 0.14355739951133728,
+ "rewards/true_env_reward_fn/mean": 0.44316986203193665,
+ "rewards/true_env_reward_fn/std": 0.14355739951133728,
+ "step": 286,
+ "step_time": 9.546786461999545
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 458.0,
+ "completions/max_terminated_length": 458.0,
+ "completions/mean_length": 77.97917175292969,
+ "completions/mean_terminated_length": 77.97917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3698437809944153,
+ "epoch": 7.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05576475337147713,
+ "kl": 9.964485252567101e-05,
+ "learning_rate": 2.536956000836082e-07,
+ "loss": 0.08634226769208908,
+ "num_tokens": 7082840.0,
+ "reward": 0.5030691027641296,
+ "reward_std": 0.2275215983390808,
+ "rewards/true_env_reward_fn/mean": 0.5030691027641296,
+ "rewards/true_env_reward_fn/std": 0.22752158343791962,
+ "step": 287,
+ "step_time": 26.703723129000537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 58.1875,
+ "completions/mean_terminated_length": 58.1875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2532718479633331,
+ "epoch": 7.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08579511940479279,
+ "kl": 9.319775290350663e-05,
+ "learning_rate": 2.500000000000001e-07,
+ "loss": -0.013113420456647873,
+ "num_tokens": 7098689.0,
+ "reward": 0.5958684682846069,
+ "reward_std": 0.1863170713186264,
+ "rewards/true_env_reward_fn/mean": 0.5958684682846069,
+ "rewards/true_env_reward_fn/std": 0.1863170564174652,
+ "step": 288,
+ "step_time": 7.618657231000725
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.20833587646484,
+ "completions/mean_terminated_length": 68.20833587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.1685407161712646,
+ "epoch": 7.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07076086848974228,
+ "kl": 7.435419411194744e-05,
+ "learning_rate": 2.4632252100977564e-07,
+ "loss": 0.010196422226727009,
+ "num_tokens": 7126387.0,
+ "reward": 0.47137731313705444,
+ "reward_std": 0.20232117176055908,
+ "rewards/true_env_reward_fn/mean": 0.47137728333473206,
+ "rewards/true_env_reward_fn/std": 0.20232117176055908,
+ "step": 289,
+ "step_time": 10.741382757999418
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2131870985031128,
+ "epoch": 7.073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07972414791584015,
+ "kl": 0.00010809541072376305,
+ "learning_rate": 2.426634296726955e-07,
+ "loss": 0.07707639783620834,
+ "num_tokens": 7150947.0,
+ "reward": 0.4951697587966919,
+ "reward_std": 0.31705158948898315,
+ "rewards/true_env_reward_fn/mean": 0.4951697587966919,
+ "rewards/true_env_reward_fn/std": 0.31705158948898315,
+ "step": 290,
+ "step_time": 11.908707627000695
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 66.41667175292969,
+ "completions/mean_terminated_length": 66.41667175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2247934639453888,
+ "epoch": 7.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06208934634923935,
+ "kl": 8.92497146196547e-05,
+ "learning_rate": 2.390229912157033e-07,
+ "loss": 0.021278446540236473,
+ "num_tokens": 7174671.0,
+ "reward": 0.44533461332321167,
+ "reward_std": 0.20755091309547424,
+ "rewards/true_env_reward_fn/mean": 0.4453345835208893,
+ "rewards/true_env_reward_fn/std": 0.20755092799663544,
+ "step": 291,
+ "step_time": 11.859711304000484
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.25,
+ "completions/mean_terminated_length": 66.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.3344177305698395,
+ "epoch": 7.121951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08249509334564209,
+ "kl": 0.00013599474277725676,
+ "learning_rate": 2.3540146951369966e-07,
+ "loss": 0.05746981129050255,
+ "num_tokens": 7202291.0,
+ "reward": 0.36803489923477173,
+ "reward_std": 0.22159868478775024,
+ "rewards/true_env_reward_fn/mean": 0.36803486943244934,
+ "rewards/true_env_reward_fn/std": 0.22159868478775024,
+ "step": 292,
+ "step_time": 9.794866193998132
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 172.0,
+ "completions/max_terminated_length": 172.0,
+ "completions/mean_length": 64.02083587646484,
+ "completions/mean_terminated_length": 64.02083587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2605153024196625,
+ "epoch": 7.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07226194441318512,
+ "kl": 7.86567807153915e-05,
+ "learning_rate": 2.3179912707041666e-07,
+ "loss": -0.06701754778623581,
+ "num_tokens": 7221060.0,
+ "reward": 0.6259939670562744,
+ "reward_std": 0.30331701040267944,
+ "rewards/true_env_reward_fn/mean": 0.6259939074516296,
+ "rewards/true_env_reward_fn/std": 0.30331701040267944,
+ "step": 293,
+ "step_time": 13.536596455999643
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 76.83333587646484,
+ "completions/mean_terminated_length": 76.83333587646484,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.275952398777008,
+ "epoch": 7.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07967559248209,
+ "kl": 0.0001376894815621199,
+ "learning_rate": 2.2821622499938948e-07,
+ "loss": -0.024285754188895226,
+ "num_tokens": 7251156.0,
+ "reward": 0.3353421688079834,
+ "reward_std": 0.26816248893737793,
+ "rewards/true_env_reward_fn/mean": 0.3353421688079834,
+ "rewards/true_env_reward_fn/std": 0.26816248893737793,
+ "step": 294,
+ "step_time": 19.762229363001097
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.4140338003635406,
+ "epoch": 7.195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07337357848882675,
+ "kl": 0.00012654263991862535,
+ "learning_rate": 2.2465302300503008e-07,
+ "loss": -0.04723845049738884,
+ "num_tokens": 7277361.0,
+ "reward": 0.4155184030532837,
+ "reward_std": 0.23990045487880707,
+ "rewards/true_env_reward_fn/mean": 0.4155184030532837,
+ "rewards/true_env_reward_fn/std": 0.23990046977996826,
+ "step": 295,
+ "step_time": 11.263231479000751
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 63.9375,
+ "completions/mean_terminated_length": 63.9375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.22734934091568,
+ "epoch": 7.219512195121951,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0908711850643158,
+ "kl": 0.00016623977535346057,
+ "learning_rate": 2.2110977936380287e-07,
+ "loss": -0.03445049002766609,
+ "num_tokens": 7298998.0,
+ "reward": 0.564518392086029,
+ "reward_std": 0.30392351746559143,
+ "rewards/true_env_reward_fn/mean": 0.564518392086029,
+ "rewards/true_env_reward_fn/std": 0.30392348766326904,
+ "step": 296,
+ "step_time": 10.9702629049998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.312496393918991,
+ "epoch": 7.2439024390243905,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08918203413486481,
+ "kl": 0.00018221777827420738,
+ "learning_rate": 2.1758675090550328e-07,
+ "loss": 0.023428799584507942,
+ "num_tokens": 7320725.0,
+ "reward": 0.4695018231868744,
+ "reward_std": 0.2527172565460205,
+ "rewards/true_env_reward_fn/mean": 0.4695018231868744,
+ "rewards/true_env_reward_fn/std": 0.2527172565460205,
+ "step": 297,
+ "step_time": 12.689384352000161
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 62.0,
+ "completions/mean_terminated_length": 62.0,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.260593295097351,
+ "epoch": 7.2682926829268295,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07724963873624802,
+ "kl": 4.775456409333856e-05,
+ "learning_rate": 2.1408419299464242e-07,
+ "loss": 0.03472111374139786,
+ "num_tokens": 7345397.0,
+ "reward": 0.39238378405570984,
+ "reward_std": 0.23648974299430847,
+ "rewards/true_env_reward_fn/mean": 0.39238378405570984,
+ "rewards/true_env_reward_fn/std": 0.23648977279663086,
+ "step": 298,
+ "step_time": 10.398283558999537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.272550791501999,
+ "epoch": 7.2926829268292686,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0737687349319458,
+ "kl": 7.711273838140187e-05,
+ "learning_rate": 2.1060235951193578e-07,
+ "loss": 0.010874807834625244,
+ "num_tokens": 7371231.0,
+ "reward": 0.4486481845378876,
+ "reward_std": 0.26590272784233093,
+ "rewards/true_env_reward_fn/mean": 0.4486481845378876,
+ "rewards/true_env_reward_fn/std": 0.26590269804000854,
+ "step": 299,
+ "step_time": 12.961759718001304
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3426357805728912,
+ "epoch": 7.317073170731708,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09169661998748779,
+ "kl": 0.00015296797573682852,
+ "learning_rate": 2.071415028359026e-07,
+ "loss": -0.0708877444267273,
+ "num_tokens": 7397151.0,
+ "reward": 0.5012298822402954,
+ "reward_std": 0.21363919973373413,
+ "rewards/true_env_reward_fn/mean": 0.5012298226356506,
+ "rewards/true_env_reward_fn/std": 0.21363921463489532,
+ "step": 300,
+ "step_time": 13.761256955000135
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 70.75,
+ "completions/mean_terminated_length": 70.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2883423566818237,
+ "epoch": 7.341463414634147,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06652592122554779,
+ "kl": 0.00011371383516234346,
+ "learning_rate": 2.0370187382457066e-07,
+ "loss": -0.011642830446362495,
+ "num_tokens": 7426403.0,
+ "reward": 0.3339906632900238,
+ "reward_std": 0.20955638587474823,
+ "rewards/true_env_reward_fn/mean": 0.3339906632900238,
+ "rewards/true_env_reward_fn/std": 0.20955640077590942,
+ "step": 301,
+ "step_time": 9.628323140000248
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 68.02083587646484,
+ "completions/mean_terminated_length": 68.02083587646484,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.1677636206150055,
+ "epoch": 7.365853658536586,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08921164274215698,
+ "kl": 7.750577606202569e-05,
+ "learning_rate": 2.0028372179729402e-07,
+ "loss": 0.024114318192005157,
+ "num_tokens": 7458380.0,
+ "reward": 0.3653101921081543,
+ "reward_std": 0.3042241334915161,
+ "rewards/true_env_reward_fn/mean": 0.3653101921081543,
+ "rewards/true_env_reward_fn/std": 0.3042241334915161,
+ "step": 302,
+ "step_time": 17.152215452000746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 61.04166793823242,
+ "completions/mean_terminated_length": 61.04166793823242,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1981053352355957,
+ "epoch": 7.390243902439025,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08189807087182999,
+ "kl": 0.00016033334395615384,
+ "learning_rate": 1.9688729451668111e-07,
+ "loss": 0.017046045511960983,
+ "num_tokens": 7481590.0,
+ "reward": 0.5003601312637329,
+ "reward_std": 0.3917968273162842,
+ "rewards/true_env_reward_fn/mean": 0.5003601312637329,
+ "rewards/true_env_reward_fn/std": 0.3917968273162842,
+ "step": 303,
+ "step_time": 10.428820308000468
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 64.95833587646484,
+ "completions/mean_terminated_length": 64.95833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2136133313179016,
+ "epoch": 7.414634146341464,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07148941606283188,
+ "kl": 5.305510785547085e-05,
+ "learning_rate": 1.9351283817063546e-07,
+ "loss": -0.009052902460098267,
+ "num_tokens": 7508932.0,
+ "reward": 0.3940971791744232,
+ "reward_std": 0.2546152174472809,
+ "rewards/true_env_reward_fn/mean": 0.3940971791744232,
+ "rewards/true_env_reward_fn/std": 0.25461524724960327,
+ "step": 304,
+ "step_time": 12.12407543900099
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2293521761894226,
+ "epoch": 7.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05877150595188141,
+ "kl": 0.00012415168203006033,
+ "learning_rate": 1.9016059735451156e-07,
+ "loss": -0.008430279791355133,
+ "num_tokens": 7535518.0,
+ "reward": 0.47240138053894043,
+ "reward_std": 0.2845028340816498,
+ "rewards/true_env_reward_fn/mean": 0.47240138053894043,
+ "rewards/true_env_reward_fn/std": 0.2845028340816498,
+ "step": 305,
+ "step_time": 12.694503639000686
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 61.270835876464844,
+ "completions/mean_terminated_length": 61.270835876464844,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.1673714816570282,
+ "epoch": 7.463414634146342,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07308074831962585,
+ "kl": 0.0001147145967479446,
+ "learning_rate": 1.8683081505338465e-07,
+ "loss": -0.06511729210615158,
+ "num_tokens": 7558883.0,
+ "reward": 0.4651026129722595,
+ "reward_std": 0.19122423231601715,
+ "rewards/true_env_reward_fn/mean": 0.46510258316993713,
+ "rewards/true_env_reward_fn/std": 0.19122423231601715,
+ "step": 306,
+ "step_time": 15.502204728000834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 63.6875,
+ "completions/mean_terminated_length": 63.6875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.304062157869339,
+ "epoch": 7.487804878048781,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0670589879155159,
+ "kl": 9.212431177729741e-05,
+ "learning_rate": 1.8352373262443915e-07,
+ "loss": -0.026926759630441666,
+ "num_tokens": 7578916.0,
+ "reward": 0.5716189742088318,
+ "reward_std": 0.19298586249351501,
+ "rewards/true_env_reward_fn/mean": 0.5716189742088318,
+ "rewards/true_env_reward_fn/std": 0.19298586249351501,
+ "step": 307,
+ "step_time": 8.911180752998916
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2283632457256317,
+ "epoch": 7.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06950085610151291,
+ "kl": 9.545813054501195e-05,
+ "learning_rate": 1.80239589779473e-07,
+ "loss": 0.010137543082237244,
+ "num_tokens": 7604028.0,
+ "reward": 0.5349916219711304,
+ "reward_std": 0.2091754674911499,
+ "rewards/true_env_reward_fn/mean": 0.5349915623664856,
+ "rewards/true_env_reward_fn/std": 0.2091754525899887,
+ "step": 308,
+ "step_time": 11.745030509000571
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 77.5,
+ "completions/mean_terminated_length": 77.5,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3315171897411346,
+ "epoch": 7.536585365853659,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05699850618839264,
+ "kl": 6.70248282403918e-05,
+ "learning_rate": 1.7697862456752271e-07,
+ "loss": 0.06499005854129791,
+ "num_tokens": 7628884.0,
+ "reward": 0.4583873748779297,
+ "reward_std": 0.26091766357421875,
+ "rewards/true_env_reward_fn/mean": 0.4583873748779297,
+ "rewards/true_env_reward_fn/std": 0.26091763377189636,
+ "step": 309,
+ "step_time": 13.87453935099984
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2704036831855774,
+ "epoch": 7.560975609756097,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09379290789365768,
+ "kl": 0.0001983325491892174,
+ "learning_rate": 1.7374107335760934e-07,
+ "loss": 0.09065254032611847,
+ "num_tokens": 7651991.0,
+ "reward": 0.5210780501365662,
+ "reward_std": 0.2745552361011505,
+ "rewards/true_env_reward_fn/mean": 0.5210780501365662,
+ "rewards/true_env_reward_fn/std": 0.2745552361011505,
+ "step": 310,
+ "step_time": 11.07200519900016
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 73.60417175292969,
+ "completions/mean_terminated_length": 73.60417175292969,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2427658140659332,
+ "epoch": 7.585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06011087819933891,
+ "kl": 6.545234282384627e-05,
+ "learning_rate": 1.7052717082160344e-07,
+ "loss": -0.0115435142070055,
+ "num_tokens": 7682948.0,
+ "reward": 0.3796111047267914,
+ "reward_std": 0.2212861329317093,
+ "rewards/true_env_reward_fn/mean": 0.3796111047267914,
+ "rewards/true_env_reward_fn/std": 0.22128616273403168,
+ "step": 311,
+ "step_time": 12.894371897999918
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1208404004573822,
+ "epoch": 7.609756097560975,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0799943059682846,
+ "kl": 0.0001426433700544294,
+ "learning_rate": 1.6733714991721738e-07,
+ "loss": -0.010216867551207542,
+ "num_tokens": 7704179.0,
+ "reward": 0.5524939298629761,
+ "reward_std": 0.26985135674476624,
+ "rewards/true_env_reward_fn/mean": 0.5524939298629761,
+ "rewards/true_env_reward_fn/std": 0.26985135674476624,
+ "step": 312,
+ "step_time": 8.69524126000033
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 76.72917175292969,
+ "completions/mean_terminated_length": 76.72917175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.2062768340110779,
+ "epoch": 7.634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05652271583676338,
+ "kl": 0.00010435856393087306,
+ "learning_rate": 1.6417124187111774e-07,
+ "loss": 0.033915065228939056,
+ "num_tokens": 7732974.0,
+ "reward": 0.40553492307662964,
+ "reward_std": 0.2561950087547302,
+ "rewards/true_env_reward_fn/mean": 0.40553489327430725,
+ "rewards/true_env_reward_fn/std": 0.2561950087547302,
+ "step": 313,
+ "step_time": 11.456125995000548
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 63.41666793823242,
+ "completions/mean_terminated_length": 63.41666793823242,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.2864463925361633,
+ "epoch": 7.658536585365853,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08890142291784286,
+ "kl": 0.00014215287774277385,
+ "learning_rate": 1.6102967616216617e-07,
+ "loss": 0.04210362955927849,
+ "num_tokens": 7754650.0,
+ "reward": 0.5954334139823914,
+ "reward_std": 0.3237621784210205,
+ "rewards/true_env_reward_fn/mean": 0.5954334139823914,
+ "rewards/true_env_reward_fn/std": 0.3237621784210205,
+ "step": 314,
+ "step_time": 11.561733381999147
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 69.22917175292969,
+ "completions/mean_terminated_length": 69.22917175292969,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2406023740768433,
+ "epoch": 7.682926829268292,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05483023077249527,
+ "kl": 3.3986466860369546e-05,
+ "learning_rate": 1.5791268050478483e-07,
+ "loss": -0.03052404522895813,
+ "num_tokens": 7777389.0,
+ "reward": 0.5392192602157593,
+ "reward_std": 0.21974749863147736,
+ "rewards/true_env_reward_fn/mean": 0.5392192006111145,
+ "rewards/true_env_reward_fn/std": 0.21974751353263855,
+ "step": 315,
+ "step_time": 13.48241268899983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 57.97916793823242,
+ "completions/mean_terminated_length": 57.97916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2167797982692719,
+ "epoch": 7.7073170731707314,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08746550232172012,
+ "kl": 0.00011399560207792092,
+ "learning_rate": 1.5482048083245114e-07,
+ "loss": 0.014903642237186432,
+ "num_tokens": 7807828.0,
+ "reward": 0.29947829246520996,
+ "reward_std": 0.2547810673713684,
+ "rewards/true_env_reward_fn/mean": 0.29947829246520996,
+ "rewards/true_env_reward_fn/std": 0.2547810673713684,
+ "step": 316,
+ "step_time": 10.080044547000398
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 65.3125,
+ "completions/mean_terminated_length": 65.3125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2823624312877655,
+ "epoch": 7.7317073170731705,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07846319675445557,
+ "kl": 0.00012852110194216948,
+ "learning_rate": 1.517533012813217e-07,
+ "loss": 0.05300665646791458,
+ "num_tokens": 7834227.0,
+ "reward": 0.4816446304321289,
+ "reward_std": 0.3394080102443695,
+ "rewards/true_env_reward_fn/mean": 0.4816446304321289,
+ "rewards/true_env_reward_fn/std": 0.3394079804420471,
+ "step": 317,
+ "step_time": 12.856840839001052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 264.0,
+ "completions/max_terminated_length": 264.0,
+ "completions/mean_length": 83.47917175292969,
+ "completions/mean_terminated_length": 83.47917175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3714390099048615,
+ "epoch": 7.7560975609756095,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0722746029496193,
+ "kl": 0.00011485655522847082,
+ "learning_rate": 1.4871136417398405e-07,
+ "loss": -0.009422918781638145,
+ "num_tokens": 7860362.0,
+ "reward": 0.44833892583847046,
+ "reward_std": 0.24655339121818542,
+ "rewards/true_env_reward_fn/mean": 0.44833889603614807,
+ "rewards/true_env_reward_fn/std": 0.24655337631702423,
+ "step": 318,
+ "step_time": 18.09142264499951
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 203.0,
+ "completions/max_terminated_length": 203.0,
+ "completions/mean_length": 69.10417175292969,
+ "completions/mean_terminated_length": 69.10417175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2680339813232422,
+ "epoch": 7.780487804878049,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09518000483512878,
+ "kl": 0.00010953140827041352,
+ "learning_rate": 1.4569489000334433e-07,
+ "loss": -0.09500816464424133,
+ "num_tokens": 7882799.0,
+ "reward": 0.4905685782432556,
+ "reward_std": 0.261008620262146,
+ "rewards/true_env_reward_fn/mean": 0.4905685484409332,
+ "rewards/true_env_reward_fn/std": 0.261008620262146,
+ "step": 319,
+ "step_time": 17.2091521250004
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 257.0,
+ "completions/max_terminated_length": 257.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2107920348644257,
+ "epoch": 7.804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07063736766576767,
+ "kl": 3.921870529666194e-05,
+ "learning_rate": 1.4270409741664268e-07,
+ "loss": -0.011502111330628395,
+ "num_tokens": 7901717.0,
+ "reward": 0.566825807094574,
+ "reward_std": 0.18909600377082825,
+ "rewards/true_env_reward_fn/mean": 0.566825807094574,
+ "rewards/true_env_reward_fn/std": 0.18909598886966705,
+ "step": 320,
+ "step_time": 15.649325063000106
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 66.66667175292969,
+ "completions/mean_terminated_length": 66.66667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2565636038780212,
+ "epoch": 7.829268292682927,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06063119322061539,
+ "kl": 4.300068394513801e-05,
+ "learning_rate": 1.3973920319960652e-07,
+ "loss": 0.001966973766684532,
+ "num_tokens": 7919013.0,
+ "reward": 0.6115278005599976,
+ "reward_std": 0.17882205545902252,
+ "rewards/true_env_reward_fn/mean": 0.6115277409553528,
+ "rewards/true_env_reward_fn/std": 0.17882204055786133,
+ "step": 321,
+ "step_time": 7.359487544999865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 57.083335876464844,
+ "completions/mean_terminated_length": 57.083335876464844,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2200327813625336,
+ "epoch": 7.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07409472018480301,
+ "kl": 0.00011575021017051768,
+ "learning_rate": 1.368004222607355e-07,
+ "loss": -0.047135986387729645,
+ "num_tokens": 7941137.0,
+ "reward": 0.5057475566864014,
+ "reward_std": 0.3366250991821289,
+ "rewards/true_env_reward_fn/mean": 0.5057475566864014,
+ "rewards/true_env_reward_fn/std": 0.3366251289844513,
+ "step": 322,
+ "step_time": 10.691665401999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 57.520835876464844,
+ "completions/mean_terminated_length": 57.520835876464844,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.3097383677959442,
+ "epoch": 7.878048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0950520932674408,
+ "kl": 0.00014910039135429543,
+ "learning_rate": 1.338879676157249e-07,
+ "loss": -0.018300604075193405,
+ "num_tokens": 7967298.0,
+ "reward": 0.39754772186279297,
+ "reward_std": 0.227029949426651,
+ "rewards/true_env_reward_fn/mean": 0.39754772186279297,
+ "rewards/true_env_reward_fn/std": 0.2270299643278122,
+ "step": 323,
+ "step_time": 9.646710404997975
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 165.0,
+ "completions/max_terminated_length": 165.0,
+ "completions/mean_length": 63.04166793823242,
+ "completions/mean_terminated_length": 63.04166793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3998334109783173,
+ "epoch": 7.902439024390244,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08187483251094818,
+ "kl": 0.0001343226194876479,
+ "learning_rate": 1.310020503720254e-07,
+ "loss": 0.10223554074764252,
+ "num_tokens": 7989452.0,
+ "reward": 0.4514659643173218,
+ "reward_std": 0.3259531557559967,
+ "rewards/true_env_reward_fn/mean": 0.4514659643173218,
+ "rewards/true_env_reward_fn/std": 0.3259531557559967,
+ "step": 324,
+ "step_time": 13.284335969000495
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 248.0,
+ "completions/max_terminated_length": 248.0,
+ "completions/mean_length": 78.77083587646484,
+ "completions/mean_terminated_length": 78.77083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3267191052436829,
+ "epoch": 7.926829268292683,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06648680567741394,
+ "kl": 5.966442222415935e-05,
+ "learning_rate": 1.281428797135402e-07,
+ "loss": -0.046651843935251236,
+ "num_tokens": 8011065.0,
+ "reward": 0.6008384227752686,
+ "reward_std": 0.19184507429599762,
+ "rewards/true_env_reward_fn/mean": 0.6008384227752686,
+ "rewards/true_env_reward_fn/std": 0.19184507429599762,
+ "step": 325,
+ "step_time": 18.06017050799983
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 61.54166793823242,
+ "completions/mean_terminated_length": 61.54166793823242,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.4001933634281158,
+ "epoch": 7.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10428700596094131,
+ "kl": 0.0001954599174496252,
+ "learning_rate": 1.253106628854635e-07,
+ "loss": -0.08898806571960449,
+ "num_tokens": 8042035.0,
+ "reward": 0.21636998653411865,
+ "reward_std": 0.26617059111595154,
+ "rewards/true_env_reward_fn/mean": 0.21636998653411865,
+ "rewards/true_env_reward_fn/std": 0.26617059111595154,
+ "step": 326,
+ "step_time": 9.872497149000083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 69.91667175292969,
+ "completions/mean_terminated_length": 69.91667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.198440283536911,
+ "epoch": 7.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06512034684419632,
+ "kl": 9.893041169561911e-05,
+ "learning_rate": 1.2250560517925745e-07,
+ "loss": -0.004943478852510452,
+ "num_tokens": 8066991.0,
+ "reward": 0.5199041962623596,
+ "reward_std": 0.2916473150253296,
+ "rewards/true_env_reward_fn/mean": 0.5199041962623596,
+ "rewards/true_env_reward_fn/std": 0.2916473150253296,
+ "step": 327,
+ "step_time": 13.27990607999891
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.3655813038349152,
+ "epoch": 8.0,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08734066039323807,
+ "kl": 0.00011680843272188213,
+ "learning_rate": 1.197279099177731e-07,
+ "loss": -0.027742527425289154,
+ "num_tokens": 8095161.0,
+ "reward": 0.4403582811355591,
+ "reward_std": 0.20052418112754822,
+ "rewards/true_env_reward_fn/mean": 0.4403582811355591,
+ "rewards/true_env_reward_fn/std": 0.20052418112754822,
+ "step": 328,
+ "step_time": 9.091917235000437
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 63.645835876464844,
+ "completions/mean_terminated_length": 63.645835876464844,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1942758560180664,
+ "epoch": 8.024390243902438,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07103214412927628,
+ "kl": 9.241796033165883e-05,
+ "learning_rate": 1.1697777844051104e-07,
+ "loss": -0.024517521262168884,
+ "num_tokens": 8117752.0,
+ "reward": 0.4961467981338501,
+ "reward_std": 0.2681204676628113,
+ "rewards/true_env_reward_fn/mean": 0.4961467981338501,
+ "rewards/true_env_reward_fn/std": 0.2681204676628113,
+ "step": 329,
+ "step_time": 10.042522196999926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 71.58333587646484,
+ "completions/mean_terminated_length": 71.58333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.209133356809616,
+ "epoch": 8.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06019354239106178,
+ "kl": 2.6403215088066645e-05,
+ "learning_rate": 1.142554100890285e-07,
+ "loss": -0.03160044550895691,
+ "num_tokens": 8140164.0,
+ "reward": 0.5655641555786133,
+ "reward_std": 0.1564219743013382,
+ "rewards/true_env_reward_fn/mean": 0.5655641555786133,
+ "rewards/true_env_reward_fn/std": 0.1564219743013382,
+ "step": 330,
+ "step_time": 8.212663780001094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 62.47916793823242,
+ "completions/mean_terminated_length": 62.47916793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.229485958814621,
+ "epoch": 8.073170731707316,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07769559323787689,
+ "kl": 0.0001446352634957293,
+ "learning_rate": 1.115610021924902e-07,
+ "loss": -0.01835213601589203,
+ "num_tokens": 8177099.0,
+ "reward": 0.1968289315700531,
+ "reward_std": 0.3009001910686493,
+ "rewards/true_env_reward_fn/mean": 0.1968289166688919,
+ "rewards/true_env_reward_fn/std": 0.3009001612663269,
+ "step": 331,
+ "step_time": 16.508294159000798
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 55.72916793823242,
+ "completions/mean_terminated_length": 55.72916793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.249097228050232,
+ "epoch": 8.097560975609756,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09750289469957352,
+ "kl": 0.00022061014351493213,
+ "learning_rate": 1.0889475005336446e-07,
+ "loss": 0.012823417782783508,
+ "num_tokens": 8202310.0,
+ "reward": 0.4960649907588959,
+ "reward_std": 0.28122010827064514,
+ "rewards/true_env_reward_fn/mean": 0.4960649907588959,
+ "rewards/true_env_reward_fn/std": 0.28122007846832275,
+ "step": 332,
+ "step_time": 9.309349606999604
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 326.0,
+ "completions/mean_length": 82.35417175292969,
+ "completions/mean_terminated_length": 73.21276092529297,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3090683817863464,
+ "epoch": 8.121951219512194,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09015674889087677,
+ "kl": 0.00010220101285085548,
+ "learning_rate": 1.0625684693326725e-07,
+ "loss": 0.23226313292980194,
+ "num_tokens": 8227663.0,
+ "reward": 0.4449918270111084,
+ "reward_std": 0.24675849080085754,
+ "rewards/true_env_reward_fn/mean": 0.4449918270111084,
+ "rewards/true_env_reward_fn/std": 0.24675849080085754,
+ "step": 333,
+ "step_time": 29.62071167800059
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 67.52083587646484,
+ "completions/mean_terminated_length": 67.52083587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2623873353004456,
+ "epoch": 8.146341463414634,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09357088804244995,
+ "kl": 0.00020532654161797836,
+ "learning_rate": 1.0364748403895368e-07,
+ "loss": 0.015600509941577911,
+ "num_tokens": 8251736.0,
+ "reward": 0.49391698837280273,
+ "reward_std": 0.313683420419693,
+ "rewards/true_env_reward_fn/mean": 0.49391698837280273,
+ "rewards/true_env_reward_fn/std": 0.3136834502220154,
+ "step": 334,
+ "step_time": 14.77577510500032
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 89.25,
+ "completions/mean_terminated_length": 89.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.3438844978809357,
+ "epoch": 8.170731707317072,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07505634427070618,
+ "kl": 8.66996942932019e-05,
+ "learning_rate": 1.0106685050845837e-07,
+ "loss": -0.006955280434340239,
+ "num_tokens": 8283596.0,
+ "reward": 0.37073618173599243,
+ "reward_std": 0.22015252709388733,
+ "rewards/true_env_reward_fn/mean": 0.37073615193367004,
+ "rewards/true_env_reward_fn/std": 0.22015254199504852,
+ "step": 335,
+ "step_time": 14.388228956000603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 196.0,
+ "completions/max_terminated_length": 196.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3619366586208344,
+ "epoch": 8.195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07524604350328445,
+ "kl": 5.828110352013027e-05,
+ "learning_rate": 9.851513339738626e-08,
+ "loss": 0.02218097448348999,
+ "num_tokens": 8304798.0,
+ "reward": 0.6131325960159302,
+ "reward_std": 0.12095426768064499,
+ "rewards/true_env_reward_fn/mean": 0.6131325364112854,
+ "rewards/true_env_reward_fn/std": 0.12095426768064499,
+ "step": 336,
+ "step_time": 12.145640587000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 145.0,
+ "completions/mean_length": 76.77083587646484,
+ "completions/mean_terminated_length": 67.51063537597656,
+ "completions/min_length": 4.0,
+ "completions/min_terminated_length": 4.0,
+ "entropy": 1.6089566349983215,
+ "epoch": 8.21951219512195,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07641154527664185,
+ "kl": 0.00011577324767131358,
+ "learning_rate": 9.599251766535343e-08,
+ "loss": -0.047279007732868195,
+ "num_tokens": 8331715.0,
+ "reward": 0.37190374732017517,
+ "reward_std": 0.2730160653591156,
+ "rewards/true_env_reward_fn/mean": 0.37190374732017517,
+ "rewards/true_env_reward_fn/std": 0.2730160653591156,
+ "step": 337,
+ "step_time": 33.08387189299992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 158.0,
+ "completions/max_terminated_length": 158.0,
+ "completions/mean_length": 60.79166793823242,
+ "completions/mean_terminated_length": 60.79166793823242,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2664433717727661,
+ "epoch": 8.24390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10079872608184814,
+ "kl": 0.00016995913574646693,
+ "learning_rate": 9.349918616258113e-08,
+ "loss": 0.04466569796204567,
+ "num_tokens": 8357801.0,
+ "reward": 0.4597291648387909,
+ "reward_std": 0.2547009587287903,
+ "rewards/true_env_reward_fn/mean": 0.4597291648387909,
+ "rewards/true_env_reward_fn/std": 0.2547009289264679,
+ "step": 338,
+ "step_time": 12.540567380000539
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 278.0,
+ "completions/max_terminated_length": 278.0,
+ "completions/mean_length": 72.95833587646484,
+ "completions/mean_terminated_length": 72.95833587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.20101597905159,
+ "epoch": 8.268292682926829,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0724605992436409,
+ "kl": 8.693387826497201e-05,
+ "learning_rate": 9.103531961664118e-08,
+ "loss": -0.04201607406139374,
+ "num_tokens": 8391047.0,
+ "reward": 0.272682249546051,
+ "reward_std": 0.22370731830596924,
+ "rewards/true_env_reward_fn/mean": 0.27268221974372864,
+ "rewards/true_env_reward_fn/std": 0.22370733320713043,
+ "step": 339,
+ "step_time": 21.499952529999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 66.27083587646484,
+ "completions/mean_terminated_length": 66.27083587646484,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.240132600069046,
+ "epoch": 8.292682926829269,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09206120669841766,
+ "kl": 9.196988685289398e-05,
+ "learning_rate": 8.860109661935672e-08,
+ "loss": 0.013995083048939705,
+ "num_tokens": 8412060.0,
+ "reward": 0.5260962843894958,
+ "reward_std": 0.16053001582622528,
+ "rewards/true_env_reward_fn/mean": 0.5260962843894958,
+ "rewards/true_env_reward_fn/std": 0.16053001582622528,
+ "step": 340,
+ "step_time": 11.287615471999743
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 187.0,
+ "completions/max_terminated_length": 187.0,
+ "completions/mean_length": 69.85417175292969,
+ "completions/mean_terminated_length": 69.85417175292969,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.2434665858745575,
+ "epoch": 8.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06606078892946243,
+ "kl": 0.00013524454698199406,
+ "learning_rate": 8.619669361385662e-08,
+ "loss": 0.04330434650182724,
+ "num_tokens": 8435189.0,
+ "reward": 0.5729736089706421,
+ "reward_std": 0.28772154450416565,
+ "rewards/true_env_reward_fn/mean": 0.5729735493659973,
+ "rewards/true_env_reward_fn/std": 0.28772154450416565,
+ "step": 341,
+ "step_time": 14.554103153000142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 87.83333587646484,
+ "completions/mean_terminated_length": 87.83333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.193087100982666,
+ "epoch": 8.341463414634147,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.06404980272054672,
+ "kl": 7.958761125337332e-05,
+ "learning_rate": 8.382228488178638e-08,
+ "loss": -0.02654789574444294,
+ "num_tokens": 8461485.0,
+ "reward": 0.4391756057739258,
+ "reward_std": 0.3396260142326355,
+ "rewards/true_env_reward_fn/mean": 0.4391756057739258,
+ "rewards/true_env_reward_fn/std": 0.3396260142326355,
+ "step": 342,
+ "step_time": 16.93046250899988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 66.0625,
+ "completions/mean_terminated_length": 66.0625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2198111712932587,
+ "epoch": 8.365853658536585,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06700363755226135,
+ "kl": 3.631071058407542e-05,
+ "learning_rate": 8.14780425306758e-08,
+ "loss": -0.01248769462108612,
+ "num_tokens": 8477128.0,
+ "reward": 0.6880009174346924,
+ "reward_std": 0.16646623611450195,
+ "rewards/true_env_reward_fn/mean": 0.6880009174346924,
+ "rewards/true_env_reward_fn/std": 0.16646623611450195,
+ "step": 343,
+ "step_time": 8.379146702999606
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 240.0,
+ "completions/max_terminated_length": 240.0,
+ "completions/mean_length": 72.0625,
+ "completions/mean_terminated_length": 72.0625,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.365002065896988,
+ "epoch": 8.390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07387524098157883,
+ "kl": 7.906068458396476e-05,
+ "learning_rate": 7.91641364814628e-08,
+ "loss": -0.019427858293056488,
+ "num_tokens": 8502659.0,
+ "reward": 0.49987101554870605,
+ "reward_std": 0.22534418106079102,
+ "rewards/true_env_reward_fn/mean": 0.49987101554870605,
+ "rewards/true_env_reward_fn/std": 0.22534416615962982,
+ "step": 344,
+ "step_time": 16.447989433999282
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 63.833335876464844,
+ "completions/mean_terminated_length": 63.833335876464844,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3033632636070251,
+ "epoch": 8.414634146341463,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07896492630243301,
+ "kl": 0.00015640226229152177,
+ "learning_rate": 7.688073445617798e-08,
+ "loss": 0.032075606286525726,
+ "num_tokens": 8527771.0,
+ "reward": 0.4142627418041229,
+ "reward_std": 0.39965084195137024,
+ "rewards/true_env_reward_fn/mean": 0.4142627418041229,
+ "rewards/true_env_reward_fn/std": 0.3996508717536926,
+ "step": 345,
+ "step_time": 10.625701701998878
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 222.0,
+ "completions/max_terminated_length": 222.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2410016655921936,
+ "epoch": 8.439024390243903,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10202007740736008,
+ "kl": 0.00028172876773169264,
+ "learning_rate": 7.462800196578661e-08,
+ "loss": 0.030375666916370392,
+ "num_tokens": 8557085.0,
+ "reward": 0.3401162028312683,
+ "reward_std": 0.2810371518135071,
+ "rewards/true_env_reward_fn/mean": 0.3401161730289459,
+ "rewards/true_env_reward_fn/std": 0.2810371518135071,
+ "step": 346,
+ "step_time": 15.773468264999792
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 62.97916793823242,
+ "completions/mean_terminated_length": 62.97916793823242,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4290962517261505,
+ "epoch": 8.463414634146341,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06221809610724449,
+ "kl": 9.756034887686837e-05,
+ "learning_rate": 7.240610229819195e-08,
+ "loss": -0.009350121021270752,
+ "num_tokens": 8583644.0,
+ "reward": 0.42270374298095703,
+ "reward_std": 0.19876612722873688,
+ "rewards/true_env_reward_fn/mean": 0.42270374298095703,
+ "rewards/true_env_reward_fn/std": 0.19876612722873688,
+ "step": 347,
+ "step_time": 10.18261290999908
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 361.0,
+ "completions/max_terminated_length": 361.0,
+ "completions/mean_length": 68.89583587646484,
+ "completions/mean_terminated_length": 68.89583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2232879400253296,
+ "epoch": 8.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07711587101221085,
+ "kl": 7.56127965360065e-05,
+ "learning_rate": 7.021519650639951e-08,
+ "loss": -0.06937319040298462,
+ "num_tokens": 8604055.0,
+ "reward": 0.5483574867248535,
+ "reward_std": 0.2410479635000229,
+ "rewards/true_env_reward_fn/mean": 0.5483574867248535,
+ "rewards/true_env_reward_fn/std": 0.2410479635000229,
+ "step": 348,
+ "step_time": 20.953229555999314
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 166.0,
+ "completions/max_terminated_length": 166.0,
+ "completions/mean_length": 74.375,
+ "completions/mean_terminated_length": 74.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.2409018576145172,
+ "epoch": 8.512195121951219,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08634211868047714,
+ "kl": 0.000148440574776032,
+ "learning_rate": 6.805544339684293e-08,
+ "loss": 0.05703897029161453,
+ "num_tokens": 8636849.0,
+ "reward": 0.2619777023792267,
+ "reward_std": 0.3443836271762848,
+ "rewards/true_env_reward_fn/mean": 0.2619777023792267,
+ "rewards/true_env_reward_fn/std": 0.3443836271762848,
+ "step": 349,
+ "step_time": 13.73512468699937
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 225.0,
+ "completions/max_terminated_length": 225.0,
+ "completions/mean_length": 64.85417175292969,
+ "completions/mean_terminated_length": 64.85417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2963348627090454,
+ "epoch": 8.536585365853659,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09569360315799713,
+ "kl": 0.0001818603413994424,
+ "learning_rate": 6.592699951787362e-08,
+ "loss": 0.0750311091542244,
+ "num_tokens": 8668938.0,
+ "reward": 0.2865946292877197,
+ "reward_std": 0.21787859499454498,
+ "rewards/true_env_reward_fn/mean": 0.2865946292877197,
+ "rewards/true_env_reward_fn/std": 0.21787859499454498,
+ "step": 350,
+ "step_time": 16.08931997000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 62.66666793823242,
+ "completions/mean_terminated_length": 62.66666793823242,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.2852083146572113,
+ "epoch": 8.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08515980839729309,
+ "kl": 0.00022268274369707797,
+ "learning_rate": 6.383001914841252e-08,
+ "loss": -0.003520023077726364,
+ "num_tokens": 8686442.0,
+ "reward": 0.5473253726959229,
+ "reward_std": 0.2312658727169037,
+ "rewards/true_env_reward_fn/mean": 0.5473253726959229,
+ "rewards/true_env_reward_fn/std": 0.2312658578157425,
+ "step": 351,
+ "step_time": 9.42530540899952
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 58.60416793823242,
+ "completions/mean_terminated_length": 58.60416793823242,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1837059557437897,
+ "epoch": 8.585365853658537,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08372796326875687,
+ "kl": 7.27423093849211e-05,
+ "learning_rate": 6.176465428676819e-08,
+ "loss": 0.003352940082550049,
+ "num_tokens": 8705967.0,
+ "reward": 0.5402672290802002,
+ "reward_std": 0.2537318766117096,
+ "rewards/true_env_reward_fn/mean": 0.5402672290802002,
+ "rewards/true_env_reward_fn/std": 0.253731906414032,
+ "step": 352,
+ "step_time": 13.651303137998184
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 66.8125,
+ "completions/mean_terminated_length": 66.8125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3596981465816498,
+ "epoch": 8.609756097560975,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09085690230131149,
+ "kl": 0.00015744064694445115,
+ "learning_rate": 5.973105463961864e-08,
+ "loss": -0.03212148696184158,
+ "num_tokens": 8733134.0,
+ "reward": 0.4029428958892822,
+ "reward_std": 0.2143087089061737,
+ "rewards/true_env_reward_fn/mean": 0.4029428958892822,
+ "rewards/true_env_reward_fn/std": 0.2143087238073349,
+ "step": 353,
+ "step_time": 12.830564498999593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.225245863199234,
+ "epoch": 8.634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11574450135231018,
+ "kl": 0.00027416441298555583,
+ "learning_rate": 5.772936761116026e-08,
+ "loss": 0.05132449418306351,
+ "num_tokens": 8761296.0,
+ "reward": 0.35531648993492126,
+ "reward_std": 0.20710644125938416,
+ "rewards/true_env_reward_fn/mean": 0.35531648993492126,
+ "rewards/true_env_reward_fn/std": 0.20710645616054535,
+ "step": 354,
+ "step_time": 11.43733696199888
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 59.72916793823242,
+ "completions/mean_terminated_length": 59.72916793823242,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1999734044075012,
+ "epoch": 8.658536585365853,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08676803112030029,
+ "kl": 0.0001471151099394774,
+ "learning_rate": 5.575973829242364e-08,
+ "loss": -0.007696744054555893,
+ "num_tokens": 8786723.0,
+ "reward": 0.3953551948070526,
+ "reward_std": 0.3271177411079407,
+ "rewards/true_env_reward_fn/mean": 0.3953551948070526,
+ "rewards/true_env_reward_fn/std": 0.32711780071258545,
+ "step": 355,
+ "step_time": 10.52069242699963
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 64.14583587646484,
+ "completions/mean_terminated_length": 64.14583587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2520077526569366,
+ "epoch": 8.682926829268293,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08498971164226532,
+ "kl": 0.00012545567187771667,
+ "learning_rate": 5.382230945075556e-08,
+ "loss": 0.034280166029930115,
+ "num_tokens": 8809690.0,
+ "reward": 0.42336568236351013,
+ "reward_std": 0.2602991759777069,
+ "rewards/true_env_reward_fn/mean": 0.42336568236351013,
+ "rewards/true_env_reward_fn/std": 0.2602991759777069,
+ "step": 356,
+ "step_time": 11.943708853999851
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 67.45833587646484,
+ "completions/mean_terminated_length": 67.45833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1441007107496262,
+ "epoch": 8.707317073170731,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05060715973377228,
+ "kl": 9.935822527040727e-05,
+ "learning_rate": 5.191722151947225e-08,
+ "loss": 0.002798471599817276,
+ "num_tokens": 8836960.0,
+ "reward": 0.39054369926452637,
+ "reward_std": 0.20666222274303436,
+ "rewards/true_env_reward_fn/mean": 0.39054369926452637,
+ "rewards/true_env_reward_fn/std": 0.20666222274303436,
+ "step": 357,
+ "step_time": 9.9503519280006
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 74.625,
+ "completions/mean_terminated_length": 65.31914520263672,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2771258652210236,
+ "epoch": 8.731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07117093354463577,
+ "kl": 0.00011358128540450707,
+ "learning_rate": 5.004461258767872e-08,
+ "loss": -0.1029033362865448,
+ "num_tokens": 8867398.0,
+ "reward": 0.46532535552978516,
+ "reward_std": 0.16995222866535187,
+ "rewards/true_env_reward_fn/mean": 0.46532535552978516,
+ "rewards/true_env_reward_fn/std": 0.16995222866535187,
+ "step": 358,
+ "step_time": 36.7129373509988
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3237419724464417,
+ "epoch": 8.75609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08377347886562347,
+ "kl": 0.00010999429287039675,
+ "learning_rate": 4.820461839026046e-08,
+ "loss": 0.05175921320915222,
+ "num_tokens": 8889378.0,
+ "reward": 0.5536255240440369,
+ "reward_std": 0.26559942960739136,
+ "rewards/true_env_reward_fn/mean": 0.5536255240440369,
+ "rewards/true_env_reward_fn/std": 0.26559942960739136,
+ "step": 359,
+ "step_time": 11.28046775799885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 74.10417175292969,
+ "completions/mean_terminated_length": 74.10417175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.3808394372463226,
+ "epoch": 8.78048780487805,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08288726955652237,
+ "kl": 0.0001290821719521773,
+ "learning_rate": 4.639737229804402e-08,
+ "loss": -0.007075890898704529,
+ "num_tokens": 8910959.0,
+ "reward": 0.5237338542938232,
+ "reward_std": 0.27443575859069824,
+ "rewards/true_env_reward_fn/mean": 0.5237338542938232,
+ "rewards/true_env_reward_fn/std": 0.27443575859069824,
+ "step": 360,
+ "step_time": 13.746602035998876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 66.95833587646484,
+ "completions/mean_terminated_length": 66.95833587646484,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2947768867015839,
+ "epoch": 8.804878048780488,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.048931583762168884,
+ "kl": 7.009085675235838e-05,
+ "learning_rate": 4.462300530813024e-08,
+ "loss": 0.041338130831718445,
+ "num_tokens": 8929517.0,
+ "reward": 0.568915069103241,
+ "reward_std": 0.17454475164413452,
+ "rewards/true_env_reward_fn/mean": 0.568915069103241,
+ "rewards/true_env_reward_fn/std": 0.17454475164413452,
+ "step": 361,
+ "step_time": 8.730673061999369
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 328.0,
+ "completions/max_terminated_length": 328.0,
+ "completions/mean_length": 67.95833587646484,
+ "completions/mean_terminated_length": 67.95833587646484,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2548468708992004,
+ "epoch": 8.829268292682928,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07735498249530792,
+ "kl": 4.255911517248023e-05,
+ "learning_rate": 4.288164603439892e-08,
+ "loss": -0.0545659139752388,
+ "num_tokens": 8950043.0,
+ "reward": 0.5133440494537354,
+ "reward_std": 0.29375824332237244,
+ "rewards/true_env_reward_fn/mean": 0.5133439898490906,
+ "rewards/true_env_reward_fn/std": 0.29375824332237244,
+ "step": 362,
+ "step_time": 22.50767488800011
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2430709302425385,
+ "epoch": 8.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08263012021780014,
+ "kl": 0.00014031877253728453,
+ "learning_rate": 4.117342069818602e-08,
+ "loss": 0.045923154801130295,
+ "num_tokens": 8972927.0,
+ "reward": 0.5072638988494873,
+ "reward_std": 0.20991377532482147,
+ "rewards/true_env_reward_fn/mean": 0.5072638988494873,
+ "rewards/true_env_reward_fn/std": 0.20991379022598267,
+ "step": 363,
+ "step_time": 14.428374270999484
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 55.29166793823242,
+ "completions/mean_terminated_length": 55.29166793823242,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.120811641216278,
+ "epoch": 8.878048780487806,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08290416747331619,
+ "kl": 0.00016724429042369593,
+ "learning_rate": 3.949845311913491e-08,
+ "loss": -0.00451173260807991,
+ "num_tokens": 8988781.0,
+ "reward": 0.6316336393356323,
+ "reward_std": 0.22976641356945038,
+ "rewards/true_env_reward_fn/mean": 0.6316335797309875,
+ "rewards/true_env_reward_fn/std": 0.22976639866828918,
+ "step": 364,
+ "step_time": 7.905863810999108
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 119.0,
+ "completions/max_terminated_length": 119.0,
+ "completions/mean_length": 68.08333587646484,
+ "completions/mean_terminated_length": 68.08333587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1943224370479584,
+ "epoch": 8.902439024390244,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07621228694915771,
+ "kl": 6.38052747490292e-05,
+ "learning_rate": 3.785686470622118e-08,
+ "loss": -0.0022124722599983215,
+ "num_tokens": 9007073.0,
+ "reward": 0.5937620401382446,
+ "reward_std": 0.1718287616968155,
+ "rewards/true_env_reward_fn/mean": 0.5937620401382446,
+ "rewards/true_env_reward_fn/std": 0.1718287467956543,
+ "step": 365,
+ "step_time": 9.652277058999061
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 57.5625,
+ "completions/mean_terminated_length": 57.5625,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2130553424358368,
+ "epoch": 8.926829268292684,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07400088012218475,
+ "kl": 0.00011410296792746522,
+ "learning_rate": 3.624877444895269e-08,
+ "loss": 0.01366148516535759,
+ "num_tokens": 9032908.0,
+ "reward": 0.4561588168144226,
+ "reward_std": 0.31542423367500305,
+ "rewards/true_env_reward_fn/mean": 0.4561587870121002,
+ "rewards/true_env_reward_fn/std": 0.31542423367500305,
+ "step": 366,
+ "step_time": 10.32316389099924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 77.25,
+ "completions/mean_terminated_length": 77.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3448135256767273,
+ "epoch": 8.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07419808954000473,
+ "kl": 9.254619635612471e-05,
+ "learning_rate": 3.467429890874424e-08,
+ "loss": 0.017107762396335602,
+ "num_tokens": 9058768.0,
+ "reward": 0.5096973180770874,
+ "reward_std": 0.15345345437526703,
+ "rewards/true_env_reward_fn/mean": 0.5096972584724426,
+ "rewards/true_env_reward_fn/std": 0.15345345437526703,
+ "step": 367,
+ "step_time": 14.400456866999775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 251.0,
+ "completions/max_terminated_length": 251.0,
+ "completions/mean_length": 76.75,
+ "completions/mean_terminated_length": 76.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.329552710056305,
+ "epoch": 8.975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.057363200932741165,
+ "kl": 9.755073915584944e-05,
+ "learning_rate": 3.3133552210468874e-08,
+ "loss": 0.008944882079958916,
+ "num_tokens": 9085804.0,
+ "reward": 0.3852631747722626,
+ "reward_std": 0.22340475022792816,
+ "rewards/true_env_reward_fn/mean": 0.3852631747722626,
+ "rewards/true_env_reward_fn/std": 0.22340476512908936,
+ "step": 368,
+ "step_time": 17.320541074000175
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 185.0,
+ "completions/max_terminated_length": 185.0,
+ "completions/mean_length": 65.66667175292969,
+ "completions/mean_terminated_length": 65.66667175292969,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2402052581310272,
+ "epoch": 9.0,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08145138621330261,
+ "kl": 0.00013890985428588465,
+ "learning_rate": 3.162664603418608e-08,
+ "loss": 0.0042404308915138245,
+ "num_tokens": 9109756.0,
+ "reward": 0.5699147582054138,
+ "reward_std": 0.2426392138004303,
+ "rewards/true_env_reward_fn/mean": 0.5699147582054138,
+ "rewards/true_env_reward_fn/std": 0.2426392287015915,
+ "step": 369,
+ "step_time": 14.103951319000771
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 64.5,
+ "completions/mean_terminated_length": 64.5,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.234388917684555,
+ "epoch": 9.024390243902438,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08930502086877823,
+ "kl": 0.00010845370161405299,
+ "learning_rate": 3.015368960704584e-08,
+ "loss": -0.007921114563941956,
+ "num_tokens": 9128316.0,
+ "reward": 0.6464049220085144,
+ "reward_std": 0.2783248722553253,
+ "rewards/true_env_reward_fn/mean": 0.6464049220085144,
+ "rewards/true_env_reward_fn/std": 0.2783248722553253,
+ "step": 370,
+ "step_time": 13.493940920000568
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 138.0,
+ "completions/max_terminated_length": 138.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3062849640846252,
+ "epoch": 9.048780487804878,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05373607575893402,
+ "kl": 9.319536547991447e-05,
+ "learning_rate": 2.8714789695372054e-08,
+ "loss": 0.003740280866622925,
+ "num_tokens": 9149964.0,
+ "reward": 0.5325896739959717,
+ "reward_std": 0.2599954307079315,
+ "rewards/true_env_reward_fn/mean": 0.5325896739959717,
+ "rewards/true_env_reward_fn/std": 0.2599954307079315,
+ "step": 371,
+ "step_time": 9.884774114998436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 68.14583587646484,
+ "completions/mean_terminated_length": 68.14583587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2394476234912872,
+ "epoch": 9.073170731707316,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07394114136695862,
+ "kl": 0.0001138780153269181,
+ "learning_rate": 2.731005059692332e-08,
+ "loss": -0.0023280810564756393,
+ "num_tokens": 9170259.0,
+ "reward": 0.5012303590774536,
+ "reward_std": 0.2768687903881073,
+ "rewards/true_env_reward_fn/mean": 0.5012302994728088,
+ "rewards/true_env_reward_fn/std": 0.2768687605857849,
+ "step": 372,
+ "step_time": 9.664578052998877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 65.25,
+ "completions/mean_terminated_length": 65.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2490745186805725,
+ "epoch": 9.097560975609756,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06408056616783142,
+ "kl": 5.751862272518338e-05,
+ "learning_rate": 2.5939574133333308e-08,
+ "loss": -0.013112705200910568,
+ "num_tokens": 9197423.0,
+ "reward": 0.42906099557876587,
+ "reward_std": 0.29850679636001587,
+ "rewards/true_env_reward_fn/mean": 0.4290609657764435,
+ "rewards/true_env_reward_fn/std": 0.29850679636001587,
+ "step": 373,
+ "step_time": 10.526233300999593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 83.47917175292969,
+ "completions/mean_terminated_length": 83.47917175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.333191066980362,
+ "epoch": 9.121951219512194,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0637340396642685,
+ "kl": 0.00014988034035923192,
+ "learning_rate": 2.4603459642729864e-08,
+ "loss": 0.008196331560611725,
+ "num_tokens": 9224102.0,
+ "reward": 0.4708556830883026,
+ "reward_std": 0.2077488750219345,
+ "rewards/true_env_reward_fn/mean": 0.4708556830883026,
+ "rewards/true_env_reward_fn/std": 0.2077488750219345,
+ "step": 374,
+ "step_time": 13.534195267000541
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 272.0,
+ "completions/max_terminated_length": 272.0,
+ "completions/mean_length": 67.77083587646484,
+ "completions/mean_terminated_length": 67.77083587646484,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.151135116815567,
+ "epoch": 9.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07668020576238632,
+ "kl": 3.856326316054037e-05,
+ "learning_rate": 2.3301803972534728e-08,
+ "loss": 0.05261360481381416,
+ "num_tokens": 9252339.0,
+ "reward": 0.379050612449646,
+ "reward_std": 0.23492643237113953,
+ "rewards/true_env_reward_fn/mean": 0.379050612449646,
+ "rewards/true_env_reward_fn/std": 0.23492641746997833,
+ "step": 375,
+ "step_time": 24.720070157999544
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.322300672531128,
+ "epoch": 9.170731707317072,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11152477562427521,
+ "kl": 0.00033667498792055994,
+ "learning_rate": 2.2034701472443852e-08,
+ "loss": -0.038920704275369644,
+ "num_tokens": 9270227.0,
+ "reward": 0.5742875933647156,
+ "reward_std": 0.2614876925945282,
+ "rewards/true_env_reward_fn/mean": 0.5742875933647156,
+ "rewards/true_env_reward_fn/std": 0.2614877223968506,
+ "step": 376,
+ "step_time": 9.753906144999746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 152.0,
+ "completions/max_terminated_length": 152.0,
+ "completions/mean_length": 67.3125,
+ "completions/mean_terminated_length": 67.3125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.322103351354599,
+ "epoch": 9.195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06792955845594406,
+ "kl": 0.00011458320295787416,
+ "learning_rate": 2.0802243987588064e-08,
+ "loss": 0.08347739279270172,
+ "num_tokens": 9299602.0,
+ "reward": 0.4416275918483734,
+ "reward_std": 0.31829217076301575,
+ "rewards/true_env_reward_fn/mean": 0.4416275918483734,
+ "rewards/true_env_reward_fn/std": 0.31829220056533813,
+ "step": 377,
+ "step_time": 13.209823182000036
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 216.0,
+ "completions/max_terminated_length": 216.0,
+ "completions/mean_length": 63.6875,
+ "completions/mean_terminated_length": 63.6875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3833276331424713,
+ "epoch": 9.21951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08396119624376297,
+ "kl": 0.00019161509044351988,
+ "learning_rate": 1.9604520851876194e-08,
+ "loss": 0.01698162779211998,
+ "num_tokens": 9331507.0,
+ "reward": 0.26609930396080017,
+ "reward_std": 0.25199094414711,
+ "rewards/true_env_reward_fn/mean": 0.26609930396080017,
+ "rewards/true_env_reward_fn/std": 0.2519909739494324,
+ "step": 378,
+ "step_time": 15.5939898680017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 62.645835876464844,
+ "completions/mean_terminated_length": 62.645835876464844,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.287186712026596,
+ "epoch": 9.24390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0844208225607872,
+ "kl": 6.250779006222729e-05,
+ "learning_rate": 1.8441618881519184e-08,
+ "loss": -0.0004980191588401794,
+ "num_tokens": 9352346.0,
+ "reward": 0.6220773458480835,
+ "reward_std": 0.17538678646087646,
+ "rewards/true_env_reward_fn/mean": 0.6220772862434387,
+ "rewards/true_env_reward_fn/std": 0.17538677155971527,
+ "step": 379,
+ "step_time": 11.331410343000243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 130.0,
+ "completions/max_terminated_length": 130.0,
+ "completions/mean_length": 65.22917175292969,
+ "completions/mean_terminated_length": 65.22917175292969,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.3174393773078918,
+ "epoch": 9.268292682926829,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08529601246118546,
+ "kl": 0.0001577844341227319,
+ "learning_rate": 1.7313622368738013e-08,
+ "loss": 0.02950325235724449,
+ "num_tokens": 9381101.0,
+ "reward": 0.3922676742076874,
+ "reward_std": 0.2629895806312561,
+ "rewards/true_env_reward_fn/mean": 0.3922676742076874,
+ "rewards/true_env_reward_fn/std": 0.2629896104335785,
+ "step": 380,
+ "step_time": 11.785110086000714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 147.0,
+ "completions/max_terminated_length": 147.0,
+ "completions/mean_length": 55.833335876464844,
+ "completions/mean_terminated_length": 55.833335876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2294963598251343,
+ "epoch": 9.292682926829269,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08615893125534058,
+ "kl": 0.00012760807658196427,
+ "learning_rate": 1.62206130756532e-08,
+ "loss": 0.03380218520760536,
+ "num_tokens": 9402133.0,
+ "reward": 0.48932167887687683,
+ "reward_std": 0.3009134531021118,
+ "rewards/true_env_reward_fn/mean": 0.48932167887687683,
+ "rewards/true_env_reward_fn/std": 0.3009134531021118,
+ "step": 381,
+ "step_time": 11.71825248299865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 112.0,
+ "completions/max_terminated_length": 112.0,
+ "completions/mean_length": 70.3125,
+ "completions/mean_terminated_length": 70.3125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2500975131988525,
+ "epoch": 9.317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08025979995727539,
+ "kl": 0.00015432167128892615,
+ "learning_rate": 1.51626702283586e-08,
+ "loss": -0.026728281751275063,
+ "num_tokens": 9433948.0,
+ "reward": 0.3480832576751709,
+ "reward_std": 0.27214208245277405,
+ "rewards/true_env_reward_fn/mean": 0.3480832576751709,
+ "rewards/true_env_reward_fn/std": 0.27214211225509644,
+ "step": 382,
+ "step_time": 11.077961950000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 207.0,
+ "completions/max_terminated_length": 207.0,
+ "completions/mean_length": 77.83333587646484,
+ "completions/mean_terminated_length": 77.83333587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2940633893013,
+ "epoch": 9.341463414634147,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07732700556516647,
+ "kl": 0.00013818908701068722,
+ "learning_rate": 1.4139870511178765e-08,
+ "loss": 0.012543938122689724,
+ "num_tokens": 9457780.0,
+ "reward": 0.4787862300872803,
+ "reward_std": 0.2881976068019867,
+ "rewards/true_env_reward_fn/mean": 0.4787862300872803,
+ "rewards/true_env_reward_fn/std": 0.2881976068019867,
+ "step": 383,
+ "step_time": 14.209161775999746
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 65.64583587646484,
+ "completions/mean_terminated_length": 65.64583587646484,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.25718092918396,
+ "epoch": 9.365853658536585,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.10170208662748337,
+ "kl": 0.00013350013250601478,
+ "learning_rate": 1.3152288061110517e-08,
+ "loss": -0.013931870460510254,
+ "num_tokens": 9475411.0,
+ "reward": 0.6825136542320251,
+ "reward_std": 0.21768879890441895,
+ "rewards/true_env_reward_fn/mean": 0.6825136542320251,
+ "rewards/true_env_reward_fn/std": 0.21768878400325775,
+ "step": 384,
+ "step_time": 8.900575762000699
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 206.0,
+ "completions/max_terminated_length": 206.0,
+ "completions/mean_length": 67.83333587646484,
+ "completions/mean_terminated_length": 67.83333587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2512764632701874,
+ "epoch": 9.390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06906364113092422,
+ "kl": 5.4121877838042565e-05,
+ "learning_rate": 1.2199994462448904e-08,
+ "loss": -0.01152697205543518,
+ "num_tokens": 9492995.0,
+ "reward": 0.5913000106811523,
+ "reward_std": 0.24868597090244293,
+ "rewards/true_env_reward_fn/mean": 0.5913000106811523,
+ "rewards/true_env_reward_fn/std": 0.24868597090244293,
+ "step": 385,
+ "step_time": 13.630191889000344
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 253.0,
+ "completions/max_terminated_length": 253.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.348086029291153,
+ "epoch": 9.414634146341463,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06427548080682755,
+ "kl": 6.0526374909386504e-05,
+ "learning_rate": 1.128305874159896e-08,
+ "loss": -0.039815668016672134,
+ "num_tokens": 9516749.0,
+ "reward": 0.48740649223327637,
+ "reward_std": 0.29916781187057495,
+ "rewards/true_env_reward_fn/mean": 0.48740649223327637,
+ "rewards/true_env_reward_fn/std": 0.29916781187057495,
+ "step": 386,
+ "step_time": 17.153756488999534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.64583587646484,
+ "completions/mean_terminated_length": 75.64583587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3448026180267334,
+ "epoch": 9.439024390243903,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.04732977971434593,
+ "kl": 5.01183408232464e-05,
+ "learning_rate": 1.0401547362071938e-08,
+ "loss": -0.03205295279622078,
+ "num_tokens": 9545300.0,
+ "reward": 0.43718665838241577,
+ "reward_std": 0.16248689591884613,
+ "rewards/true_env_reward_fn/mean": 0.4371866285800934,
+ "rewards/true_env_reward_fn/std": 0.16248688101768494,
+ "step": 387,
+ "step_time": 11.892045039999175
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 57.458335876464844,
+ "completions/mean_terminated_length": 57.458335876464844,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1685180962085724,
+ "epoch": 9.463414634146341,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0918080136179924,
+ "kl": 0.00025482301134616137,
+ "learning_rate": 9.555524219667988e-09,
+ "loss": 0.0153394415974617,
+ "num_tokens": 9568906.0,
+ "reward": 0.46309399604797363,
+ "reward_std": 0.27685311436653137,
+ "rewards/true_env_reward_fn/mean": 0.46309399604797363,
+ "rewards/true_env_reward_fn/std": 0.276853084564209,
+ "step": 388,
+ "step_time": 9.755774155000836
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 167.0,
+ "completions/max_terminated_length": 167.0,
+ "completions/mean_length": 62.520835876464844,
+ "completions/mean_terminated_length": 62.520835876464844,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.268288493156433,
+ "epoch": 9.487804878048781,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09713966399431229,
+ "kl": 0.00021595886028080713,
+ "learning_rate": 8.745050637844532e-09,
+ "loss": 0.10685908794403076,
+ "num_tokens": 9596667.0,
+ "reward": 0.375690221786499,
+ "reward_std": 0.27006518840789795,
+ "rewards/true_env_reward_fn/mean": 0.375690221786499,
+ "rewards/true_env_reward_fn/std": 0.27006518840789795,
+ "step": 389,
+ "step_time": 13.222017186999437
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 239.0,
+ "completions/max_terminated_length": 239.0,
+ "completions/mean_length": 69.20833587646484,
+ "completions/mean_terminated_length": 69.20833587646484,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2847715020179749,
+ "epoch": 9.512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08115239441394806,
+ "kl": 0.0001902375151985325,
+ "learning_rate": 7.970185363271431e-09,
+ "loss": -0.015931159257888794,
+ "num_tokens": 9620285.0,
+ "reward": 0.5215861797332764,
+ "reward_std": 0.29267051815986633,
+ "rewards/true_env_reward_fn/mean": 0.5215861797332764,
+ "rewards/true_env_reward_fn/std": 0.29267051815986633,
+ "step": 390,
+ "step_time": 19.701098540999737
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 75.70833587646484,
+ "completions/mean_terminated_length": 75.70833587646484,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.1211588382720947,
+ "epoch": 9.536585365853659,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.050371747463941574,
+ "kl": 5.3559015668724896e-05,
+ "learning_rate": 7.230984561572729e-09,
+ "loss": -0.027369298040866852,
+ "num_tokens": 9643879.0,
+ "reward": 0.5257008075714111,
+ "reward_std": 0.31278616189956665,
+ "rewards/true_env_reward_fn/mean": 0.5257008075714111,
+ "rewards/true_env_reward_fn/std": 0.31278616189956665,
+ "step": 391,
+ "step_time": 14.5430295999995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 73.22917175292969,
+ "completions/mean_terminated_length": 73.22917175292969,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.162130355834961,
+ "epoch": 9.560975609756097,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06951762735843658,
+ "kl": 0.00011329071639920585,
+ "learning_rate": 6.5275018132553425e-09,
+ "loss": 0.005165033042430878,
+ "num_tokens": 9674170.0,
+ "reward": 0.38153380155563354,
+ "reward_std": 0.2553096115589142,
+ "rewards/true_env_reward_fn/mean": 0.38153377175331116,
+ "rewards/true_env_reward_fn/std": 0.2553096115589142,
+ "step": 392,
+ "step_time": 13.884568534998834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 64.60417175292969,
+ "completions/mean_terminated_length": 64.60417175292969,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.1723724007606506,
+ "epoch": 9.585365853658537,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06391289085149765,
+ "kl": 0.00012617898028111085,
+ "learning_rate": 5.859788109825792e-09,
+ "loss": -0.026335250586271286,
+ "num_tokens": 9697471.0,
+ "reward": 0.5592976808547974,
+ "reward_std": 0.294358491897583,
+ "rewards/true_env_reward_fn/mean": 0.5592976212501526,
+ "rewards/true_env_reward_fn/std": 0.294358491897583,
+ "step": 393,
+ "step_time": 10.88813568399928
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 75.0625,
+ "completions/mean_terminated_length": 75.0625,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2699165344238281,
+ "epoch": 9.609756097560975,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0821971669793129,
+ "kl": 0.00011502238521643449,
+ "learning_rate": 5.2278918500933134e-09,
+ "loss": 0.020743347704410553,
+ "num_tokens": 9727482.0,
+ "reward": 0.3525441288948059,
+ "reward_std": 0.24436385929584503,
+ "rewards/true_env_reward_fn/mean": 0.3525440990924835,
+ "rewards/true_env_reward_fn/std": 0.24436385929584503,
+ "step": 394,
+ "step_time": 13.137662936999732
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 64.14583587646484,
+ "completions/mean_terminated_length": 64.14583587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2177735269069672,
+ "epoch": 9.634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08585982769727707,
+ "kl": 9.731890622788342e-05,
+ "learning_rate": 4.631858836662561e-09,
+ "loss": -0.0022293110378086567,
+ "num_tokens": 9752601.0,
+ "reward": 0.42186734080314636,
+ "reward_std": 0.14065590500831604,
+ "rewards/true_env_reward_fn/mean": 0.42186734080314636,
+ "rewards/true_env_reward_fn/std": 0.14065590500831604,
+ "step": 395,
+ "step_time": 9.551430144000733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 72.95833587646484,
+ "completions/mean_terminated_length": 72.95833587646484,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3700671792030334,
+ "epoch": 9.658536585365853,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0687054842710495,
+ "kl": 0.00012583125862875022,
+ "learning_rate": 4.071732272613148e-09,
+ "loss": -0.010379405692219734,
+ "num_tokens": 9776159.0,
+ "reward": 0.40437865257263184,
+ "reward_std": 0.20715568959712982,
+ "rewards/true_env_reward_fn/mean": 0.40437865257263184,
+ "rewards/true_env_reward_fn/std": 0.20715567469596863,
+ "step": 396,
+ "step_time": 9.576685387999532
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 65.0625,
+ "completions/mean_terminated_length": 65.0625,
+ "completions/min_length": 22.0,
+ "completions/min_terminated_length": 22.0,
+ "entropy": 1.1619998216629028,
+ "epoch": 9.682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06019222363829613,
+ "kl": 8.781285851000575e-05,
+ "learning_rate": 3.5475527583680997e-09,
+ "loss": -0.023738304153084755,
+ "num_tokens": 9797522.0,
+ "reward": 0.5591601729393005,
+ "reward_std": 0.2965308129787445,
+ "rewards/true_env_reward_fn/mean": 0.5591601729393005,
+ "rewards/true_env_reward_fn/std": 0.2965308427810669,
+ "step": 397,
+ "step_time": 12.962693579000188
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 66.39583587646484,
+ "completions/mean_terminated_length": 56.91489028930664,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.0172521024942398,
+ "epoch": 9.707317073170731,
+ "frac_reward_zero_std": 0.8333333730697632,
+ "grad_norm": 0.02889176644384861,
+ "kl": 7.828448997315718e-05,
+ "learning_rate": 3.0593582887512014e-09,
+ "loss": 0.11688737571239471,
+ "num_tokens": 9817717.0,
+ "reward": 0.5341037511825562,
+ "reward_std": 0.33641698956489563,
+ "rewards/true_env_reward_fn/mean": 0.5341036915779114,
+ "rewards/true_env_reward_fn/std": 0.33641698956489563,
+ "step": 398,
+ "step_time": 30.514281355000094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 383.0,
+ "completions/max_terminated_length": 383.0,
+ "completions/mean_length": 70.72917175292969,
+ "completions/mean_terminated_length": 70.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2152343690395355,
+ "epoch": 9.731707317073171,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07666711509227753,
+ "kl": 0.00014014396219863556,
+ "learning_rate": 2.6071842502326526e-09,
+ "loss": -0.051372602581977844,
+ "num_tokens": 9847608.0,
+ "reward": 0.46223634481430054,
+ "reward_std": 0.21379408240318298,
+ "rewards/true_env_reward_fn/mean": 0.46223631501197815,
+ "rewards/true_env_reward_fn/std": 0.21379409730434418,
+ "step": 399,
+ "step_time": 24.57003485899986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 156.0,
+ "completions/max_terminated_length": 156.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.272740215063095,
+ "epoch": 9.75609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06314582377672195,
+ "kl": 0.0001023452778099454,
+ "learning_rate": 2.1910634183644472e-09,
+ "loss": 0.0021672993898391724,
+ "num_tokens": 9869730.0,
+ "reward": 0.41130340099334717,
+ "reward_std": 0.35641229152679443,
+ "rewards/true_env_reward_fn/mean": 0.41130340099334717,
+ "rewards/true_env_reward_fn/std": 0.3564123213291168,
+ "step": 400,
+ "step_time": 12.93115923699952
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 187.0,
+ "completions/max_terminated_length": 187.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2798932790756226,
+ "epoch": 9.78048780487805,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07237259298563004,
+ "kl": 0.00013262193169794045,
+ "learning_rate": 1.8110259554043327e-09,
+ "loss": 0.0007664486765861511,
+ "num_tokens": 9892131.0,
+ "reward": 0.47880715131759644,
+ "reward_std": 0.2620767652988434,
+ "rewards/true_env_reward_fn/mean": 0.47880712151527405,
+ "rewards/true_env_reward_fn/std": 0.2620767652988434,
+ "step": 401,
+ "step_time": 13.1478075240002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 56.75,
+ "completions/mean_terminated_length": 56.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1527983248233795,
+ "epoch": 9.804878048780488,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.09068363159894943,
+ "kl": 0.0002173124230466783,
+ "learning_rate": 1.4670994081297795e-09,
+ "loss": 0.031802937388420105,
+ "num_tokens": 9916919.0,
+ "reward": 0.4450836181640625,
+ "reward_std": 0.28454098105430603,
+ "rewards/true_env_reward_fn/mean": 0.4450836181640625,
+ "rewards/true_env_reward_fn/std": 0.2845410406589508,
+ "step": 402,
+ "step_time": 10.531171943999652
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 177.0,
+ "completions/max_terminated_length": 177.0,
+ "completions/mean_length": 70.0625,
+ "completions/mean_terminated_length": 70.0625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.357511967420578,
+ "epoch": 9.829268292682928,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.05832836031913757,
+ "kl": 7.576237385364948e-05,
+ "learning_rate": 1.1593087058410778e-09,
+ "loss": -0.0797688439488411,
+ "num_tokens": 9944290.0,
+ "reward": 0.29735517501831055,
+ "reward_std": 0.26128143072128296,
+ "rewards/true_env_reward_fn/mean": 0.29735514521598816,
+ "rewards/true_env_reward_fn/std": 0.26128146052360535,
+ "step": 403,
+ "step_time": 13.960200211999108
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 217.0,
+ "completions/max_terminated_length": 217.0,
+ "completions/mean_length": 64.64583587646484,
+ "completions/mean_terminated_length": 64.64583587646484,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1934557557106018,
+ "epoch": 9.853658536585366,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0956556499004364,
+ "kl": 0.00020315543952165172,
+ "learning_rate": 8.876761585545068e-10,
+ "loss": 0.14966487884521484,
+ "num_tokens": 9975201.0,
+ "reward": 0.3204185366630554,
+ "reward_std": 0.22474516928195953,
+ "rewards/true_env_reward_fn/mean": 0.3204185366630554,
+ "rewards/true_env_reward_fn/std": 0.22474516928195953,
+ "step": 404,
+ "step_time": 16.59387802799938
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 72.6875,
+ "completions/mean_terminated_length": 72.6875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.300670087337494,
+ "epoch": 9.878048780487806,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.0959375649690628,
+ "kl": 0.0002827896096277982,
+ "learning_rate": 6.522214553850158e-10,
+ "loss": 0.009999219328165054,
+ "num_tokens": 10005314.0,
+ "reward": 0.26286351680755615,
+ "reward_std": 0.27993860840797424,
+ "rewards/true_env_reward_fn/mean": 0.26286348700523376,
+ "rewards/true_env_reward_fn/std": 0.27993860840797424,
+ "step": 405,
+ "step_time": 15.79836324199914
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 180.0,
+ "completions/max_terminated_length": 180.0,
+ "completions/mean_length": 62.10416793823242,
+ "completions/mean_terminated_length": 62.10416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3013499975204468,
+ "epoch": 9.902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06491809338331223,
+ "kl": 8.734427501622122e-05,
+ "learning_rate": 4.529616631193112e-10,
+ "loss": 0.03921665996313095,
+ "num_tokens": 10030607.0,
+ "reward": 0.4792874753475189,
+ "reward_std": 0.20676079392433167,
+ "rewards/true_env_reward_fn/mean": 0.4792874753475189,
+ "rewards/true_env_reward_fn/std": 0.20676080882549286,
+ "step": 406,
+ "step_time": 14.00729640100053
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.02083333395421505,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 72.70833587646484,
+ "completions/mean_terminated_length": 63.36170196533203,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4017383754253387,
+ "epoch": 9.926829268292684,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.10481218993663788,
+ "kl": 0.0001222127211804036,
+ "learning_rate": 2.8991122497862287e-10,
+ "loss": 0.35084569454193115,
+ "num_tokens": 10050313.0,
+ "reward": 0.5769625902175903,
+ "reward_std": 0.22158536314964294,
+ "rewards/true_env_reward_fn/mean": 0.5769625902175903,
+ "rewards/true_env_reward_fn/std": 0.22158534824848175,
+ "step": 407,
+ "step_time": 28.694236055998772
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 66.47917175292969,
+ "completions/mean_terminated_length": 66.47917175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1504006087779999,
+ "epoch": 9.951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07713646441698074,
+ "kl": 0.00011729796096915379,
+ "learning_rate": 1.6308195957182026e-10,
+ "loss": 0.018286442384123802,
+ "num_tokens": 10076048.0,
+ "reward": 0.4973433017730713,
+ "reward_std": 0.26644790172576904,
+ "rewards/true_env_reward_fn/mean": 0.4973433017730713,
+ "rewards/true_env_reward_fn/std": 0.26644790172576904,
+ "step": 408,
+ "step_time": 10.989900497000235
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 68.4375,
+ "completions/mean_terminated_length": 68.4375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2830024063587189,
+ "epoch": 9.975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07968083769083023,
+ "kl": 3.46645133504353e-05,
+ "learning_rate": 7.248306003865279e-11,
+ "loss": 0.06483273953199387,
+ "num_tokens": 10095621.0,
+ "reward": 0.5615556836128235,
+ "reward_std": 0.12372879683971405,
+ "rewards/true_env_reward_fn/mean": 0.5615556836128235,
+ "rewards/true_env_reward_fn/std": 0.12372878938913345,
+ "step": 409,
+ "step_time": 9.636585925000873
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2528311908245087,
+ "epoch": 10.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.06754950433969498,
+ "kl": 2.3663689717068337e-05,
+ "learning_rate": 1.8121093383671738e-11,
+ "loss": 0.045541826635599136,
+ "num_tokens": 10121507.0,
+ "reward": 0.3544771373271942,
+ "reward_std": 0.22188176214694977,
+ "rewards/true_env_reward_fn/mean": 0.3544771373271942,
+ "rewards/true_env_reward_fn/std": 0.22188177704811096,
+ "step": 410,
+ "step_time": 13.209858957999131
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 410,
+ "num_input_tokens_seen": 10121507,
+ "num_train_epochs": 10,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 12,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-410/training_args.bin b/checkpoint-410/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..2276397b5f715b1fed46fa8d458d3bde360f7b81
--- /dev/null
+++ b/checkpoint-410/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
+size 6776
diff --git a/checkpoint-50/adapter_model.safetensors b/checkpoint-50/adapter_model.safetensors
index 41f4d0bcb31c19b0b40b39b0cea452fef7435087..e5092b1eab254f4796def2a8a6ea7cbc8ff987b4 100644
--- a/checkpoint-50/adapter_model.safetensors
+++ b/checkpoint-50/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:c29d16da19e145b7fbc421d2926b57f8a64fb43aa41d6a7e0e650b362d91bdcc
+oid sha256:da05d014ced353f7444520473733d53c8e7fdfabfece43d22a117a0c5fff7894
size 8731128
diff --git a/checkpoint-50/optimizer.pt b/checkpoint-50/optimizer.pt
index 624e64865fe18a4876906054cacb75458b072bd9..4e4a8523ac316e27798261b053808552768b2681 100644
--- a/checkpoint-50/optimizer.pt
+++ b/checkpoint-50/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:f36ff735dcde2bd473a8af37048e6d1ef9de616653181fc0aedf95971b4e311b
+oid sha256:390c24780cada8673a2b31cc09ff5b8ccbb119410c06e54c367334c3f28aeaf2
size 17526842
diff --git a/checkpoint-50/ref/adapter_model.safetensors b/checkpoint-50/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/checkpoint-50/ref/adapter_model.safetensors
+++ b/checkpoint-50/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/checkpoint-50/rng_state.pth b/checkpoint-50/rng_state.pth
index 1255bd47b09c1c1d5e66537d92eb0be204a3d8aa..2d36f85f66fb7adc989477742cb70fa25267c17e 100644
--- a/checkpoint-50/rng_state.pth
+++ b/checkpoint-50/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:6a8029353fcc2917f4dbe146b9fe43c7d58980906ebdeb29c19d8a55b6a7e36b
+oid sha256:0a56b022fb59713cfb6f45e9a99a7270393978a6fafa887f75dcc7d6079d33ee
size 14244
diff --git a/checkpoint-50/scheduler.pt b/checkpoint-50/scheduler.pt
index a0574e7184eee77da8cca88e473915c7f5203554..86a6bd3c560e2235773dc1a373fc73886d8156f7 100644
--- a/checkpoint-50/scheduler.pt
+++ b/checkpoint-50/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:12478c219de6b77a5dc1e7fcc9ef1b62a708c4dfc483dc14a2b66e2a1ade05dd
+oid sha256:ec3ad24a99416b8beb3a33893540106a6ff207d1decc52b9c5826370a59931b9
size 1064
diff --git a/checkpoint-50/trainer_state.json b/checkpoint-50/trainer_state.json
index a9aeab3accbd70e2a3a52a958bca65aca0cae400..2af435b8b080f8c252fbcfae61f03da9e09afe9f 100644
--- a/checkpoint-50/trainer_state.json
+++ b/checkpoint-50/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 0.4065040650406504,
+ "epoch": 1.2195121951219512,
"eval_steps": 500,
"global_step": 50,
"is_hyper_param_search": false,
@@ -16,26 +16,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 56.25,
- "completions/mean_terminated_length": 56.25,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1208415031433105,
- "epoch": 0.008130081300813009,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2371738702058792,
- "kl": 1.1247546808590414e-05,
+ "completions/max_length": 131.0,
+ "completions/max_terminated_length": 131.0,
+ "completions/mean_length": 60.10416793823242,
+ "completions/mean_terminated_length": 60.10416793823242,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2584454119205475,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09302648901939392,
+ "kl": 1.2248776783962967e-05,
"learning_rate": 0.0,
- "loss": 0.019987963140010834,
- "num_tokens": 2250.0,
- "reward": 0.7777429223060608,
- "reward_std": 0.14680756628513336,
- "rewards/true_env_reward_fn/mean": 0.7777429223060608,
- "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "loss": -0.0423424206674099,
+ "num_tokens": 23029.0,
+ "reward": 0.5082165002822876,
+ "reward_std": 0.27811428904533386,
+ "rewards/true_env_reward_fn/mean": 0.5082164406776428,
+ "rewards/true_env_reward_fn/std": 0.27811428904533386,
"step": 1,
- "step_time": 3.622400252999796
+ "step_time": 11.815711200999885
},
{
"clip_ratio/high_max": 0.0,
@@ -44,26 +44,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 126.0,
- "completions/max_terminated_length": 126.0,
- "completions/mean_length": 72.875,
- "completions/mean_terminated_length": 72.875,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 0.9768376648426056,
- "epoch": 0.016260162601626018,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10255444794893265,
- "kl": 9.209406016452704e-06,
- "learning_rate": 2.1621621621621623e-08,
- "loss": -0.16146813333034515,
- "num_tokens": 8517.0,
- "reward": 0.5471514463424683,
- "reward_std": 0.19726651906967163,
- "rewards/true_env_reward_fn/mean": 0.5471514463424683,
- "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "completions/max_length": 161.0,
+ "completions/max_terminated_length": 161.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.3789870142936707,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11938872188329697,
+ "kl": 1.2672078355535632e-05,
+ "learning_rate": 2.4390243902439023e-08,
+ "loss": -0.11833255738019943,
+ "num_tokens": 57015.0,
+ "reward": 0.1327376663684845,
+ "reward_std": 0.241567462682724,
+ "rewards/true_env_reward_fn/mean": 0.1327376663684845,
+ "rewards/true_env_reward_fn/std": 0.241567462682724,
"step": 2,
- "step_time": 5.979386726001394
+ "step_time": 13.493524850000085
},
{
"clip_ratio/high_max": 0.0,
@@ -72,26 +72,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 67.125,
- "completions/mean_terminated_length": 67.125,
- "completions/min_length": 53.0,
- "completions/min_terminated_length": 53.0,
- "entropy": 1.324017882347107,
- "epoch": 0.024390243902439025,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13558730483055115,
- "kl": 1.2776082257914823e-05,
- "learning_rate": 4.3243243243243246e-08,
- "loss": 0.008048340678215027,
- "num_tokens": 11858.0,
- "reward": 0.5399026870727539,
- "reward_std": 0.04722921922802925,
- "rewards/true_env_reward_fn/mean": 0.5399026870727539,
- "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 63.79166793823242,
+ "completions/mean_terminated_length": 63.79166793823242,
+ "completions/min_length": 7.0,
+ "completions/min_terminated_length": 7.0,
+ "entropy": 1.315225213766098,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08519645780324936,
+ "kl": 1.2407871281538974e-05,
+ "learning_rate": 4.878048780487805e-08,
+ "loss": -0.03654177859425545,
+ "num_tokens": 86989.0,
+ "reward": 0.3152047097682953,
+ "reward_std": 0.3069385886192322,
+ "rewards/true_env_reward_fn/mean": 0.3152047097682953,
+ "rewards/true_env_reward_fn/std": 0.30693864822387695,
"step": 3,
- "step_time": 3.6805073480009014
+ "step_time": 11.449303891999875
},
{
"clip_ratio/high_max": 0.0,
@@ -100,26 +100,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 53.625,
- "completions/mean_terminated_length": 53.625,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "entropy": 1.0729783773422241,
- "epoch": 0.032520325203252036,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23012493550777435,
- "kl": 1.0804946214193478e-05,
- "learning_rate": 6.486486486486487e-08,
- "loss": 0.13091428577899933,
- "num_tokens": 15379.0,
- "reward": 0.4351762533187866,
- "reward_std": 0.2320314645767212,
- "rewards/true_env_reward_fn/mean": 0.4351762533187866,
- "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "completions/max_length": 226.0,
+ "completions/max_terminated_length": 226.0,
+ "completions/mean_length": 77.20833587646484,
+ "completions/mean_terminated_length": 77.20833587646484,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.338063895702362,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08080132305622101,
+ "kl": 1.239982589140709e-05,
+ "learning_rate": 7.317073170731706e-08,
+ "loss": 0.053779490292072296,
+ "num_tokens": 112007.0,
+ "reward": 0.4893929362297058,
+ "reward_std": 0.28476035594940186,
+ "rewards/true_env_reward_fn/mean": 0.4893929064273834,
+ "rewards/true_env_reward_fn/std": 0.28476035594940186,
"step": 4,
- "step_time": 3.421140036001816
+ "step_time": 18.835909622000145
},
{
"clip_ratio/high_max": 0.0,
@@ -128,26 +128,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 57.25,
- "completions/mean_terminated_length": 57.25,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.3000869154930115,
- "epoch": 0.04065040650406504,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22867721319198608,
- "kl": 1.2170262834843015e-05,
- "learning_rate": 8.648648648648649e-08,
- "loss": 0.08851668983697891,
- "num_tokens": 19401.0,
- "reward": 0.4031979441642761,
- "reward_std": 0.32033228874206543,
- "rewards/true_env_reward_fn/mean": 0.4031979441642761,
- "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "completions/max_length": 212.0,
+ "completions/max_terminated_length": 212.0,
+ "completions/mean_length": 67.41667175292969,
+ "completions/mean_terminated_length": 67.41667175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3855182826519012,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08783729374408722,
+ "kl": 1.1660237760224845e-05,
+ "learning_rate": 9.75609756097561e-08,
+ "loss": -0.026884621009230614,
+ "num_tokens": 135883.0,
+ "reward": 0.48575252294540405,
+ "reward_std": 0.335994690656662,
+ "rewards/true_env_reward_fn/mean": 0.48575249314308167,
+ "rewards/true_env_reward_fn/std": 0.335994690656662,
"step": 5,
- "step_time": 3.977350764000221
+ "step_time": 14.435845696000001
},
{
"clip_ratio/high_max": 0.0,
@@ -156,26 +156,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 58.5,
- "completions/mean_terminated_length": 58.5,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "entropy": 1.1719728112220764,
- "epoch": 0.04878048780487805,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16202858090400696,
- "kl": 1.228428209287813e-05,
- "learning_rate": 1.0810810810810811e-07,
- "loss": 0.1666201949119568,
- "num_tokens": 21253.0,
- "reward": 0.5827490091323853,
- "reward_std": 0.27126544713974,
- "rewards/true_env_reward_fn/mean": 0.5827490091323853,
- "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "completions/max_length": 164.0,
+ "completions/max_terminated_length": 164.0,
+ "completions/mean_length": 71.29167175292969,
+ "completions/mean_terminated_length": 71.29167175292969,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2962585091590881,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.08510823547840118,
+ "kl": 1.241418908648484e-05,
+ "learning_rate": 1.219512195121951e-07,
+ "loss": -0.05353507027029991,
+ "num_tokens": 157537.0,
+ "reward": 0.47622889280319214,
+ "reward_std": 0.3605790138244629,
+ "rewards/true_env_reward_fn/mean": 0.47622886300086975,
+ "rewards/true_env_reward_fn/std": 0.3605790138244629,
"step": 6,
- "step_time": 4.179320960000041
+ "step_time": 13.232063896999989
},
{
"clip_ratio/high_max": 0.0,
@@ -184,26 +184,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.3112086653709412,
- "epoch": 0.056910569105691054,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20938768982887268,
- "kl": 1.306734156969469e-05,
- "learning_rate": 1.2972972972972974e-07,
- "loss": 0.04748187214136124,
- "num_tokens": 25726.0,
- "reward": 0.2716812491416931,
- "reward_std": 0.29254475235939026,
- "rewards/true_env_reward_fn/mean": 0.2716812491416931,
- "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "completions/max_length": 114.0,
+ "completions/max_terminated_length": 114.0,
+ "completions/mean_length": 69.45833587646484,
+ "completions/mean_terminated_length": 69.45833587646484,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.273663192987442,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0775279700756073,
+ "kl": 1.2900356978207128e-05,
+ "learning_rate": 1.4634146341463413e-07,
+ "loss": -0.010494321584701538,
+ "num_tokens": 179167.0,
+ "reward": 0.5062826871871948,
+ "reward_std": 0.18032674491405487,
+ "rewards/true_env_reward_fn/mean": 0.5062826871871948,
+ "rewards/true_env_reward_fn/std": 0.18032673001289368,
"step": 7,
- "step_time": 3.438178512999002
+ "step_time": 9.810652986000036
},
{
"clip_ratio/high_max": 0.0,
@@ -212,26 +212,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 78.5,
- "completions/mean_terminated_length": 78.5,
- "completions/min_length": 65.0,
- "completions/min_terminated_length": 65.0,
- "entropy": 1.2046615481376648,
- "epoch": 0.06504065040650407,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.857898890506476e-05,
- "kl": 1.3628536180476658e-05,
- "learning_rate": 1.5135135135135135e-07,
- "loss": 6.865971045044716e-07,
- "num_tokens": 30126.0,
- "reward": 0.49959999322891235,
- "reward_std": 0.02822280302643776,
- "rewards/true_env_reward_fn/mean": 0.49959999322891235,
- "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "completions/max_length": 287.0,
+ "completions/max_terminated_length": 287.0,
+ "completions/mean_length": 65.54167175292969,
+ "completions/mean_terminated_length": 65.54167175292969,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.255563884973526,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07464194297790527,
+ "kl": 1.1561841347429436e-05,
+ "learning_rate": 1.7073170731707317e-07,
+ "loss": 0.0830899029970169,
+ "num_tokens": 201865.0,
+ "reward": 0.38212963938713074,
+ "reward_std": 0.29894331097602844,
+ "rewards/true_env_reward_fn/mean": 0.38212963938713074,
+ "rewards/true_env_reward_fn/std": 0.29894331097602844,
"step": 8,
- "step_time": 4.324984626000514
+ "step_time": 19.874756868999953
},
{
"clip_ratio/high_max": 0.0,
@@ -240,26 +240,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 62.75,
- "completions/mean_terminated_length": 62.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2111859917640686,
- "epoch": 0.07317073170731707,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11552055925130844,
- "kl": 1.0166647598452982e-05,
- "learning_rate": 1.7297297297297298e-07,
- "loss": -0.03883127495646477,
- "num_tokens": 33332.0,
- "reward": 0.571246862411499,
- "reward_std": 0.2893567681312561,
- "rewards/true_env_reward_fn/mean": 0.571246862411499,
- "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 68.33333587646484,
+ "completions/mean_terminated_length": 68.33333587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2706169188022614,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.049192048609256744,
+ "kl": 1.157601468548819e-05,
+ "learning_rate": 1.951219512195122e-07,
+ "loss": 0.010864660143852234,
+ "num_tokens": 219953.0,
+ "reward": 0.6740004420280457,
+ "reward_std": 0.18809831142425537,
+ "rewards/true_env_reward_fn/mean": 0.6740004420280457,
+ "rewards/true_env_reward_fn/std": 0.18809829652309418,
"step": 9,
- "step_time": 3.651253555999574
+ "step_time": 9.458149736999985
},
{
"clip_ratio/high_max": 0.0,
@@ -268,26 +268,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 58.125,
- "completions/mean_terminated_length": 58.125,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.195803463459015,
- "epoch": 0.08130081300813008,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14317302405834198,
- "kl": 1.476421539337025e-05,
- "learning_rate": 1.945945945945946e-07,
- "loss": -0.034387920051813126,
- "num_tokens": 36381.0,
- "reward": 0.5593140125274658,
- "reward_std": 0.42223072052001953,
- "rewards/true_env_reward_fn/mean": 0.5593140125274658,
- "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "completions/max_length": 148.0,
+ "completions/max_terminated_length": 148.0,
+ "completions/mean_length": 59.833335876464844,
+ "completions/mean_terminated_length": 59.833335876464844,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.1927059888839722,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.0561092346906662,
+ "kl": 1.0622535000948119e-05,
+ "learning_rate": 2.195121951219512e-07,
+ "loss": -0.02407176047563553,
+ "num_tokens": 244913.0,
+ "reward": 0.5113257169723511,
+ "reward_std": 0.32156965136528015,
+ "rewards/true_env_reward_fn/mean": 0.5113256573677063,
+ "rewards/true_env_reward_fn/std": 0.32156962156295776,
"step": 10,
- "step_time": 3.8103441190014564
+ "step_time": 14.219840567000006
},
{
"clip_ratio/high_max": 0.0,
@@ -296,26 +296,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 61.625,
- "completions/mean_terminated_length": 61.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.389159917831421,
- "epoch": 0.08943089430894309,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 65.47917175292969,
+ "completions/mean_terminated_length": 65.47917175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2782267928123474,
+ "epoch": 0.2682926829268293,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11816789954900742,
- "kl": 1.0807365470100194e-05,
- "learning_rate": 2.1621621621621622e-07,
- "loss": 0.04055345058441162,
- "num_tokens": 41830.0,
- "reward": 0.12224999815225601,
- "reward_std": 0.27913153171539307,
- "rewards/true_env_reward_fn/mean": 0.12224999815225601,
- "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "grad_norm": 0.05816411226987839,
+ "kl": 1.2071807759639341e-05,
+ "learning_rate": 2.439024390243902e-07,
+ "loss": 0.007693461142480373,
+ "num_tokens": 269080.0,
+ "reward": 0.37106746435165405,
+ "reward_std": 0.26608046889305115,
+ "rewards/true_env_reward_fn/mean": 0.37106743454933167,
+ "rewards/true_env_reward_fn/std": 0.26608046889305115,
"step": 11,
- "step_time": 4.204996996000773
+ "step_time": 9.271131832999913
},
{
"clip_ratio/high_max": 0.0,
@@ -324,26 +324,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 106.0,
- "completions/max_terminated_length": 106.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "entropy": 1.3866143822669983,
- "epoch": 0.0975609756097561,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.19537723064422607,
- "kl": 1.5072046608111123e-05,
- "learning_rate": 2.3783783783783783e-07,
- "loss": -0.07735465466976166,
- "num_tokens": 47047.0,
- "reward": 0.3571999967098236,
- "reward_std": 0.18295250833034515,
- "rewards/true_env_reward_fn/mean": 0.3571999967098236,
- "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 67.9375,
+ "completions/mean_terminated_length": 67.9375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3190773129463196,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09140665084123611,
+ "kl": 1.2069708191120299e-05,
+ "learning_rate": 2.682926829268293e-07,
+ "loss": 0.07185906916856766,
+ "num_tokens": 291317.0,
+ "reward": 0.4376159906387329,
+ "reward_std": 0.27247554063796997,
+ "rewards/true_env_reward_fn/mean": 0.4376159906387329,
+ "rewards/true_env_reward_fn/std": 0.27247554063796997,
"step": 12,
- "step_time": 4.775358541999594
+ "step_time": 12.184364300000084
},
{
"clip_ratio/high_max": 0.0,
@@ -352,26 +352,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 55.75,
- "completions/mean_terminated_length": 55.75,
- "completions/min_length": 38.0,
- "completions/min_terminated_length": 38.0,
- "entropy": 1.1633875966072083,
- "epoch": 0.10569105691056911,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14852823317050934,
- "kl": 1.4038786503078882e-05,
- "learning_rate": 2.594594594594595e-07,
- "loss": -0.04705440253019333,
- "num_tokens": 51521.0,
- "reward": 0.44465911388397217,
- "reward_std": 0.15160730481147766,
- "rewards/true_env_reward_fn/mean": 0.44465911388397217,
- "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.54167175292969,
+ "completions/mean_terminated_length": 66.54167175292969,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.3555113077163696,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08824986964464188,
+ "kl": 1.2127976788178785e-05,
+ "learning_rate": 2.9268292682926825e-07,
+ "loss": -0.0217185840010643,
+ "num_tokens": 313623.0,
+ "reward": 0.5092746615409851,
+ "reward_std": 0.3137436807155609,
+ "rewards/true_env_reward_fn/mean": 0.5092746615409851,
+ "rewards/true_env_reward_fn/std": 0.3137436509132385,
"step": 13,
- "step_time": 4.072596639998665
+ "step_time": 10.720424850000086
},
{
"clip_ratio/high_max": 0.0,
@@ -380,26 +380,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 60.375,
- "completions/mean_terminated_length": 60.375,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.2392634153366089,
- "epoch": 0.11382113821138211,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23616985976696014,
- "kl": 1.3279905488161603e-05,
- "learning_rate": 2.810810810810811e-07,
- "loss": -0.021731968969106674,
- "num_tokens": 55556.0,
- "reward": 0.4130214750766754,
- "reward_std": 0.43705809116363525,
- "rewards/true_env_reward_fn/mean": 0.4130214750766754,
- "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "completions/max_length": 188.0,
+ "completions/max_terminated_length": 188.0,
+ "completions/mean_length": 69.3125,
+ "completions/mean_terminated_length": 69.3125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3283279240131378,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.05055573210120201,
+ "kl": 1.3128182672517141e-05,
+ "learning_rate": 3.170731707317073e-07,
+ "loss": -0.024722743779420853,
+ "num_tokens": 339118.0,
+ "reward": 0.45545920729637146,
+ "reward_std": 0.18457132577896118,
+ "rewards/true_env_reward_fn/mean": 0.45545920729637146,
+ "rewards/true_env_reward_fn/std": 0.18457134068012238,
"step": 14,
- "step_time": 3.7906999759998143
+ "step_time": 14.965493325000011
},
{
"clip_ratio/high_max": 0.0,
@@ -408,26 +408,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.1012902855873108,
- "epoch": 0.12195121951219512,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 66.45833587646484,
+ "completions/mean_terminated_length": 66.45833587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2629931271076202,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.11504171043634415,
- "kl": 1.1161580914631486e-05,
- "learning_rate": 3.027027027027027e-07,
- "loss": -0.03352496027946472,
- "num_tokens": 58644.0,
- "reward": 0.54444819688797,
- "reward_std": 0.2691938281059265,
- "rewards/true_env_reward_fn/mean": 0.54444819688797,
- "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "grad_norm": 0.06967486441135406,
+ "kl": 1.1465989928183262e-05,
+ "learning_rate": 3.4146341463414634e-07,
+ "loss": 0.046319857239723206,
+ "num_tokens": 366364.0,
+ "reward": 0.4448578357696533,
+ "reward_std": 0.24966756999492645,
+ "rewards/true_env_reward_fn/mean": 0.4448578357696533,
+ "rewards/true_env_reward_fn/std": 0.24966755509376526,
"step": 15,
- "step_time": 3.427628186998845
+ "step_time": 13.628413805999912
},
{
"clip_ratio/high_max": 0.0,
@@ -436,26 +436,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 66.125,
- "completions/mean_terminated_length": 66.125,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "entropy": 1.1984660625457764,
- "epoch": 0.13008130081300814,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12652896344661713,
- "kl": 1.292689512411016e-05,
- "learning_rate": 3.243243243243243e-07,
- "loss": -0.08787620067596436,
- "num_tokens": 62361.0,
- "reward": 0.46189582347869873,
- "reward_std": 0.23188425600528717,
- "rewards/true_env_reward_fn/mean": 0.46189582347869873,
- "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "completions/max_length": 194.0,
+ "completions/max_terminated_length": 194.0,
+ "completions/mean_length": 69.04167175292969,
+ "completions/mean_terminated_length": 69.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2793545722961426,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.04725664108991623,
+ "kl": 1.1130929124192335e-05,
+ "learning_rate": 3.6585365853658536e-07,
+ "loss": 0.006799306720495224,
+ "num_tokens": 392926.0,
+ "reward": 0.414639949798584,
+ "reward_std": 0.2748004198074341,
+ "rewards/true_env_reward_fn/mean": 0.414639949798584,
+ "rewards/true_env_reward_fn/std": 0.2748004198074341,
"step": 16,
- "step_time": 4.776189491001787
+ "step_time": 14.229579036999894
},
{
"clip_ratio/high_max": 0.0,
@@ -464,26 +464,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 58.25,
- "completions/mean_terminated_length": 58.25,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.1009634137153625,
- "epoch": 0.13821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1329507976770401,
- "kl": 1.1219160569453379e-05,
- "learning_rate": 3.4594594594594597e-07,
- "loss": 0.07219867408275604,
- "num_tokens": 65899.0,
- "reward": 0.65608811378479,
- "reward_std": 0.2155800759792328,
- "rewards/true_env_reward_fn/mean": 0.65608811378479,
- "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "completions/max_length": 195.0,
+ "completions/max_terminated_length": 195.0,
+ "completions/mean_length": 76.4375,
+ "completions/mean_terminated_length": 76.4375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3106227219104767,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06872504949569702,
+ "kl": 1.2065312830600305e-05,
+ "learning_rate": 3.902439024390244e-07,
+ "loss": 0.036527130752801895,
+ "num_tokens": 419219.0,
+ "reward": 0.49165210127830505,
+ "reward_std": 0.267509400844574,
+ "rewards/true_env_reward_fn/mean": 0.49165210127830505,
+ "rewards/true_env_reward_fn/std": 0.267509400844574,
"step": 17,
- "step_time": 3.525365152998347
+ "step_time": 17.023353198999985
},
{
"clip_ratio/high_max": 0.0,
@@ -492,26 +492,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "entropy": 1.1671696901321411,
- "epoch": 0.14634146341463414,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 71.72917175292969,
+ "completions/mean_terminated_length": 71.72917175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3780030608177185,
+ "epoch": 0.43902439024390244,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.15754961967468262,
- "kl": 1.3107276572554838e-05,
- "learning_rate": 3.6756756756756757e-07,
- "loss": -0.016363894566893578,
- "num_tokens": 70155.0,
- "reward": 0.3013000190258026,
- "reward_std": 0.2883487343788147,
- "rewards/true_env_reward_fn/mean": 0.3013000190258026,
- "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "grad_norm": 0.05453665927052498,
+ "kl": 1.2325858278927626e-05,
+ "learning_rate": 4.146341463414634e-07,
+ "loss": 0.01989848166704178,
+ "num_tokens": 442822.0,
+ "reward": 0.5288735032081604,
+ "reward_std": 0.2950553297996521,
+ "rewards/true_env_reward_fn/mean": 0.5288735032081604,
+ "rewards/true_env_reward_fn/std": 0.2950552701950073,
"step": 18,
- "step_time": 4.143123763000403
+ "step_time": 11.965533113999868
},
{
"clip_ratio/high_max": 0.0,
@@ -520,26 +520,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 70.25,
- "completions/mean_terminated_length": 70.25,
- "completions/min_length": 48.0,
- "completions/min_terminated_length": 48.0,
- "entropy": 1.266749083995819,
- "epoch": 0.15447154471544716,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11665906757116318,
- "kl": 1.2845626315538539e-05,
- "learning_rate": 3.891891891891892e-07,
- "loss": -0.11013027280569077,
- "num_tokens": 73389.0,
- "reward": 0.6058553457260132,
- "reward_std": 0.11022671312093735,
- "rewards/true_env_reward_fn/mean": 0.6058553457260132,
- "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 65.4375,
+ "completions/mean_terminated_length": 65.4375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3424750864505768,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09814280271530151,
+ "kl": 1.2686515219684225e-05,
+ "learning_rate": 4.390243902439024e-07,
+ "loss": 0.06940581649541855,
+ "num_tokens": 467275.0,
+ "reward": 0.5175753831863403,
+ "reward_std": 0.2811976969242096,
+ "rewards/true_env_reward_fn/mean": 0.5175753235816956,
+ "rewards/true_env_reward_fn/std": 0.2811976969242096,
"step": 19,
- "step_time": 4.701202698999623
+ "step_time": 10.33812468799988
},
{
"clip_ratio/high_max": 0.0,
@@ -548,26 +548,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "entropy": 1.1111925840377808,
- "epoch": 0.16260162601626016,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1429353505373001,
- "kl": 8.694359621586045e-06,
- "learning_rate": 4.108108108108108e-07,
- "loss": -0.05066477507352829,
- "num_tokens": 77594.0,
- "reward": 0.4271581172943115,
- "reward_std": 0.050101421773433685,
- "rewards/true_env_reward_fn/mean": 0.4271581172943115,
- "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "completions/max_length": 244.0,
+ "completions/max_terminated_length": 244.0,
+ "completions/mean_length": 65.10417175292969,
+ "completions/mean_terminated_length": 65.10417175292969,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1681120097637177,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09474422037601471,
+ "kl": 1.2183225862827385e-05,
+ "learning_rate": 4.634146341463415e-07,
+ "loss": 0.05423373728990555,
+ "num_tokens": 494320.0,
+ "reward": 0.48628994822502136,
+ "reward_std": 0.25381213426589966,
+ "rewards/true_env_reward_fn/mean": 0.48628994822502136,
+ "rewards/true_env_reward_fn/std": 0.25381216406822205,
"step": 20,
- "step_time": 3.220270914998764
+ "step_time": 17.317542748000164
},
{
"clip_ratio/high_max": 0.0,
@@ -576,26 +576,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 46.0,
- "completions/mean_terminated_length": 46.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "entropy": 1.4938308596611023,
- "epoch": 0.17073170731707318,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16324248909950256,
- "kl": 1.1220067335671047e-05,
- "learning_rate": 4.3243243243243244e-07,
- "loss": 0.023503631353378296,
- "num_tokens": 83298.0,
- "reward": 0.1186770498752594,
- "reward_std": 0.16449356079101562,
- "rewards/true_env_reward_fn/mean": 0.1186770498752594,
- "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 62.395835876464844,
+ "completions/mean_terminated_length": 62.395835876464844,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2504475116729736,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0819205492734909,
+ "kl": 1.0698822279664455e-05,
+ "learning_rate": 4.878048780487804e-07,
+ "loss": 0.05607657879590988,
+ "num_tokens": 518323.0,
+ "reward": 0.4693639278411865,
+ "reward_std": 0.32881346344947815,
+ "rewards/true_env_reward_fn/mean": 0.4693639278411865,
+ "rewards/true_env_reward_fn/std": 0.32881346344947815,
"step": 21,
- "step_time": 3.451675898999383
+ "step_time": 12.20283881399996
},
{
"clip_ratio/high_max": 0.0,
@@ -604,26 +604,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 122.0,
- "completions/max_terminated_length": 122.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.2311039566993713,
- "epoch": 0.17886178861788618,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.23201963305473328,
- "kl": 1.2657743809540989e-05,
- "learning_rate": 4.54054054054054e-07,
- "loss": 0.20273712277412415,
- "num_tokens": 87825.0,
- "reward": 0.3444172441959381,
- "reward_std": 0.4508652687072754,
- "rewards/true_env_reward_fn/mean": 0.3444172441959381,
- "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "completions/max_length": 144.0,
+ "completions/max_terminated_length": 144.0,
+ "completions/mean_length": 68.91667175292969,
+ "completions/mean_terminated_length": 68.91667175292969,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2199381291866302,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06889473646879196,
+ "kl": 1.1745656820494332e-05,
+ "learning_rate": 5.121951219512195e-07,
+ "loss": -0.017973195761442184,
+ "num_tokens": 543591.0,
+ "reward": 0.49388420581817627,
+ "reward_std": 0.2952423393726349,
+ "rewards/true_env_reward_fn/mean": 0.49388420581817627,
+ "rewards/true_env_reward_fn/std": 0.2952423095703125,
"step": 22,
- "step_time": 5.440214132999245
+ "step_time": 11.211206898000114
},
{
"clip_ratio/high_max": 0.0,
@@ -632,26 +632,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 66.875,
- "completions/mean_terminated_length": 66.875,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
"completions/min_length": 41.0,
"completions/min_terminated_length": 41.0,
- "entropy": 1.3744811415672302,
- "epoch": 0.18699186991869918,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.12073361128568649,
- "kl": 1.356211032543797e-05,
- "learning_rate": 4.7567567567567566e-07,
- "loss": -0.06243758648633957,
- "num_tokens": 92940.0,
- "reward": 0.28657954931259155,
- "reward_std": 0.19488918781280518,
- "rewards/true_env_reward_fn/mean": 0.28657954931259155,
- "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "entropy": 1.2588726878166199,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08144447952508926,
+ "kl": 1.2306870758038713e-05,
+ "learning_rate": 5.365853658536586e-07,
+ "loss": 0.02826106920838356,
+ "num_tokens": 567973.0,
+ "reward": 0.48142755031585693,
+ "reward_std": 0.26756224036216736,
+ "rewards/true_env_reward_fn/mean": 0.48142755031585693,
+ "rewards/true_env_reward_fn/std": 0.26756221055984497,
"step": 23,
- "step_time": 4.085832714999924
+ "step_time": 10.428452587999914
},
{
"clip_ratio/high_max": 0.0,
@@ -660,26 +660,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 100.0,
- "completions/max_terminated_length": 100.0,
- "completions/mean_length": 68.875,
- "completions/mean_terminated_length": 68.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "entropy": 1.3229535818099976,
- "epoch": 0.1951219512195122,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.64079047460109e-05,
- "kl": 1.1639681815722724e-05,
- "learning_rate": 4.972972972972973e-07,
- "loss": 5.819025545861223e-07,
- "num_tokens": 94923.0,
- "reward": 0.7253252267837524,
- "reward_std": 0.046159788966178894,
- "rewards/true_env_reward_fn/mean": 0.7253252267837524,
- "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.384379804134369,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11170398443937302,
+ "kl": 1.2296073691686615e-05,
+ "learning_rate": 5.609756097560975e-07,
+ "loss": 0.07271970808506012,
+ "num_tokens": 590248.0,
+ "reward": 0.38166365027427673,
+ "reward_std": 0.34809473156929016,
+ "rewards/true_env_reward_fn/mean": 0.38166365027427673,
+ "rewards/true_env_reward_fn/std": 0.3480947017669678,
"step": 24,
- "step_time": 4.218084741001803
+ "step_time": 11.223491792000118
},
{
"clip_ratio/high_max": 0.0,
@@ -688,26 +688,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 113.0,
- "completions/max_terminated_length": 113.0,
- "completions/mean_length": 76.375,
- "completions/mean_terminated_length": 76.375,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.3325599431991577,
- "epoch": 0.2032520325203252,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 63.35416793823242,
+ "completions/mean_terminated_length": 63.35416793823242,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.3013385236263275,
+ "epoch": 0.6097560975609756,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.17998270690441132,
- "kl": 1.4024041774973739e-05,
- "learning_rate": 5.18918918918919e-07,
- "loss": 0.13167564570903778,
- "num_tokens": 99178.0,
- "reward": 0.44252532720565796,
- "reward_std": 0.1883804053068161,
- "rewards/true_env_reward_fn/mean": 0.44252532720565796,
- "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "grad_norm": 0.10069931298494339,
+ "kl": 1.2947949016961502e-05,
+ "learning_rate": 5.853658536585365e-07,
+ "loss": 0.033605337142944336,
+ "num_tokens": 615345.0,
+ "reward": 0.5046355724334717,
+ "reward_std": 0.2754679322242737,
+ "rewards/true_env_reward_fn/mean": 0.5046355128288269,
+ "rewards/true_env_reward_fn/std": 0.2754679322242737,
"step": 25,
- "step_time": 4.84537445100068
+ "step_time": 10.92509102200006
},
{
"clip_ratio/high_max": 0.0,
@@ -716,26 +716,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 88.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 64.375,
- "completions/mean_terminated_length": 64.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.2788519263267517,
- "epoch": 0.21138211382113822,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1275048851966858,
- "kl": 1.1262640327913687e-05,
- "learning_rate": 5.405405405405405e-07,
- "loss": -0.010535649955272675,
- "num_tokens": 102353.0,
- "reward": 0.3852383494377136,
- "reward_std": 0.2447713315486908,
- "rewards/true_env_reward_fn/mean": 0.3852383494377136,
- "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 61.41666793823242,
+ "completions/mean_terminated_length": 61.41666793823242,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2652399837970734,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07595694065093994,
+ "kl": 1.151612354988174e-05,
+ "learning_rate": 6.097560975609756e-07,
+ "loss": 0.04607678949832916,
+ "num_tokens": 644749.0,
+ "reward": 0.3311978578567505,
+ "reward_std": 0.21527718007564545,
+ "rewards/true_env_reward_fn/mean": 0.3311978578567505,
+ "rewards/true_env_reward_fn/std": 0.21527719497680664,
"step": 26,
- "step_time": 3.80895136899926
+ "step_time": 10.458724108999945
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +744,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 63.25,
- "completions/mean_terminated_length": 63.25,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.0208025872707367,
- "epoch": 0.21951219512195122,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14226751029491425,
- "kl": 1.4639559594797902e-05,
- "learning_rate": 5.621621621621622e-07,
- "loss": -0.05629514902830124,
- "num_tokens": 103867.0,
- "reward": 0.8898874521255493,
- "reward_std": 0.1414213478565216,
- "rewards/true_env_reward_fn/mean": 0.8898874521255493,
- "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.193794459104538,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.07690244168043137,
+ "kl": 1.2164698546257569e-05,
+ "learning_rate": 6.341463414634146e-07,
+ "loss": 0.00818883627653122,
+ "num_tokens": 671153.0,
+ "reward": 0.3635203242301941,
+ "reward_std": 0.23849114775657654,
+ "rewards/true_env_reward_fn/mean": 0.3635202944278717,
+ "rewards/true_env_reward_fn/std": 0.23849113285541534,
"step": 27,
- "step_time": 3.227140603999942
+ "step_time": 14.364785926000081
},
{
"clip_ratio/high_max": 0.0,
@@ -772,26 +772,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 66.5,
- "completions/mean_terminated_length": 66.5,
- "completions/min_length": 55.0,
- "completions/min_terminated_length": 55.0,
- "entropy": 1.278637707233429,
- "epoch": 0.22764227642276422,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.6319210822694e-05,
- "kl": 1.394796117892838e-05,
- "learning_rate": 5.837837837837838e-07,
- "loss": 6.984611218285863e-07,
- "num_tokens": 108511.0,
- "reward": 0.5384680032730103,
- "reward_std": 0.06977442651987076,
- "rewards/true_env_reward_fn/mean": 0.5384680032730103,
- "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 63.4375,
+ "completions/mean_terminated_length": 63.4375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2883787751197815,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.0902288407087326,
+ "kl": 1.1798915693361778e-05,
+ "learning_rate": 6.585365853658536e-07,
+ "loss": 0.038317371159791946,
+ "num_tokens": 697614.0,
+ "reward": 0.44166144728660583,
+ "reward_std": 0.25748196244239807,
+ "rewards/true_env_reward_fn/mean": 0.44166144728660583,
+ "rewards/true_env_reward_fn/std": 0.25748199224472046,
"step": 28,
- "step_time": 3.3963304120006796
+ "step_time": 10.888908384999922
},
{
"clip_ratio/high_max": 0.0,
@@ -800,26 +800,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 47.0,
- "completions/min_terminated_length": 47.0,
- "entropy": 1.2547507286071777,
- "epoch": 0.23577235772357724,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.10846269875764847,
- "kl": 1.5149432329053525e-05,
- "learning_rate": 6.054054054054054e-07,
- "loss": 0.004249647259712219,
- "num_tokens": 111323.0,
- "reward": 0.6256026029586792,
- "reward_std": 0.350762277841568,
- "rewards/true_env_reward_fn/mean": 0.6256026029586792,
- "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "completions/max_length": 238.0,
+ "completions/max_terminated_length": 238.0,
+ "completions/mean_length": 69.60417175292969,
+ "completions/mean_terminated_length": 69.60417175292969,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3002805709838867,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07522639632225037,
+ "kl": 1.2230455695316778e-05,
+ "learning_rate": 6.829268292682927e-07,
+ "loss": 0.031045034527778625,
+ "num_tokens": 719187.0,
+ "reward": 0.5349087119102478,
+ "reward_std": 0.29909756779670715,
+ "rewards/true_env_reward_fn/mean": 0.5349087119102478,
+ "rewards/true_env_reward_fn/std": 0.29909753799438477,
"step": 29,
- "step_time": 4.2103285969988065
+ "step_time": 15.510035302999995
},
{
"clip_ratio/high_max": 0.0,
@@ -828,26 +828,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 77.0,
- "completions/max_terminated_length": 77.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "entropy": 1.4687196612358093,
- "epoch": 0.24390243902439024,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13451272249221802,
- "kl": 1.3284446140460204e-05,
- "learning_rate": 6.27027027027027e-07,
- "loss": 0.05542291700839996,
- "num_tokens": 115976.0,
- "reward": 0.3901680111885071,
- "reward_std": 0.2995865046977997,
- "rewards/true_env_reward_fn/mean": 0.3901680111885071,
- "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 70.91667175292969,
+ "completions/mean_terminated_length": 70.91667175292969,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2718828916549683,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06776711344718933,
+ "kl": 1.2617916354429326e-05,
+ "learning_rate": 7.073170731707316e-07,
+ "loss": 0.09301326423883438,
+ "num_tokens": 744095.0,
+ "reward": 0.43472790718078613,
+ "reward_std": 0.3138841986656189,
+ "rewards/true_env_reward_fn/mean": 0.43472790718078613,
+ "rewards/true_env_reward_fn/std": 0.3138841688632965,
"step": 30,
- "step_time": 3.5506420210003853
+ "step_time": 14.50245602599989
},
{
"clip_ratio/high_max": 0.0,
@@ -856,26 +856,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 98.0,
- "completions/max_terminated_length": 98.0,
- "completions/mean_length": 76.875,
- "completions/mean_terminated_length": 76.875,
- "completions/min_length": 52.0,
- "completions/min_terminated_length": 52.0,
- "entropy": 1.2640270590782166,
- "epoch": 0.25203252032520324,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.18553969264030457,
- "kl": 1.2505860468081664e-05,
- "learning_rate": 6.486486486486486e-07,
- "loss": -0.015417251735925674,
- "num_tokens": 118471.0,
- "reward": 0.6587758660316467,
- "reward_std": 0.14417217671871185,
- "rewards/true_env_reward_fn/mean": 0.6587758660316467,
- "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "completions/max_length": 153.0,
+ "completions/max_terminated_length": 153.0,
+ "completions/mean_length": 69.77083587646484,
+ "completions/mean_terminated_length": 69.77083587646484,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2918945252895355,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08040682971477509,
+ "kl": 1.2672349157583085e-05,
+ "learning_rate": 7.317073170731707e-07,
+ "loss": 0.0367550291121006,
+ "num_tokens": 764612.0,
+ "reward": 0.5134401321411133,
+ "reward_std": 0.19073942303657532,
+ "rewards/true_env_reward_fn/mean": 0.5134401321411133,
+ "rewards/true_env_reward_fn/std": 0.19073940813541412,
"step": 31,
- "step_time": 4.198089399002129
+ "step_time": 11.06186091799998
},
{
"clip_ratio/high_max": 0.0,
@@ -884,26 +884,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 56.0,
- "completions/mean_terminated_length": 56.0,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 1.5262224078178406,
- "epoch": 0.2601626016260163,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.2357814759016037,
- "kl": 1.6242850506387185e-05,
- "learning_rate": 6.702702702702702e-07,
- "loss": 0.031210914254188538,
- "num_tokens": 123923.0,
- "reward": 0.0943702906370163,
- "reward_std": 0.1497660130262375,
- "rewards/true_env_reward_fn/mean": 0.0943702906370163,
- "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "completions/max_length": 269.0,
+ "completions/max_terminated_length": 269.0,
+ "completions/mean_length": 71.79167175292969,
+ "completions/mean_terminated_length": 71.79167175292969,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.1679067015647888,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0744430273771286,
+ "kl": 1.2661263326663175e-05,
+ "learning_rate": 7.560975609756097e-07,
+ "loss": 0.05885648727416992,
+ "num_tokens": 782058.0,
+ "reward": 0.5372593402862549,
+ "reward_std": 0.18350909650325775,
+ "rewards/true_env_reward_fn/mean": 0.5372593402862549,
+ "rewards/true_env_reward_fn/std": 0.18350908160209656,
"step": 32,
- "step_time": 3.978757984001277
+ "step_time": 15.808748693000211
},
{
"clip_ratio/high_max": 0.0,
@@ -912,26 +912,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 82.0,
- "completions/max_terminated_length": 82.0,
- "completions/mean_length": 59.5,
- "completions/mean_terminated_length": 59.5,
- "completions/min_length": 20.0,
- "completions/min_terminated_length": 20.0,
- "entropy": 0.9924907088279724,
- "epoch": 0.2682926829268293,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20393438637256622,
- "kl": 1.181096149593941e-05,
- "learning_rate": 6.918918918918919e-07,
- "loss": -0.0068489015102386475,
- "num_tokens": 130831.0,
- "reward": 0.20862048864364624,
- "reward_std": 0.2418184131383896,
- "rewards/true_env_reward_fn/mean": 0.20862048864364624,
- "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 76.79167175292969,
+ "completions/mean_terminated_length": 76.79167175292969,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1829756796360016,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.051698025315999985,
+ "kl": 1.0996191576850833e-05,
+ "learning_rate": 7.804878048780488e-07,
+ "loss": 0.010143717750906944,
+ "num_tokens": 810472.0,
+ "reward": 0.4369215667247772,
+ "reward_std": 0.30869919061660767,
+ "rewards/true_env_reward_fn/mean": 0.4369215667247772,
+ "rewards/true_env_reward_fn/std": 0.30869919061660767,
"step": 33,
- "step_time": 4.237411461999727
+ "step_time": 24.20358999299981
},
{
"clip_ratio/high_max": 0.0,
@@ -940,26 +940,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 184.0,
- "completions/max_terminated_length": 184.0,
- "completions/mean_length": 105.0,
- "completions/mean_terminated_length": 105.0,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.4909422397613525,
- "epoch": 0.2764227642276423,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 7.714608364040032e-05,
- "kl": 1.3747331649938133e-05,
- "learning_rate": 7.135135135135134e-07,
- "loss": 6.856024583612452e-07,
- "num_tokens": 138663.0,
- "reward": 0.1821666657924652,
- "reward_std": 0.2963036298751831,
- "rewards/true_env_reward_fn/mean": 0.1821666657924652,
- "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 61.85416793823242,
+ "completions/mean_terminated_length": 61.85416793823242,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2468958497047424,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.09706687182188034,
+ "kl": 1.2097383432774222e-05,
+ "learning_rate": 8.048780487804878e-07,
+ "loss": 0.026558157056570053,
+ "num_tokens": 836713.0,
+ "reward": 0.3587157428264618,
+ "reward_std": 0.2754887044429779,
+ "rewards/true_env_reward_fn/mean": 0.3587157428264618,
+ "rewards/true_env_reward_fn/std": 0.2754887044429779,
"step": 34,
- "step_time": 8.45711429900075
+ "step_time": 12.218407348999904
},
{
"clip_ratio/high_max": 0.0,
@@ -968,26 +968,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 61.25,
- "completions/mean_terminated_length": 61.25,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.0832659006118774,
- "epoch": 0.2845528455284553,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1297609806060791,
- "kl": 1.1829738923552213e-05,
- "learning_rate": 7.351351351351351e-07,
- "loss": -0.02754262089729309,
- "num_tokens": 142361.0,
- "reward": 0.4525124728679657,
- "reward_std": 0.23157824575901031,
- "rewards/true_env_reward_fn/mean": 0.4525124728679657,
- "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 59.5625,
+ "completions/mean_terminated_length": 59.5625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2368170320987701,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08964981138706207,
+ "kl": 1.3131634887031396e-05,
+ "learning_rate": 8.292682926829268e-07,
+ "loss": -0.01139204390347004,
+ "num_tokens": 860028.0,
+ "reward": 0.49109315872192383,
+ "reward_std": 0.20359393954277039,
+ "rewards/true_env_reward_fn/mean": 0.49109315872192383,
+ "rewards/true_env_reward_fn/std": 0.20359393954277039,
"step": 35,
- "step_time": 3.564060039998367
+ "step_time": 9.66908789599995
},
{
"clip_ratio/high_max": 0.0,
@@ -996,26 +996,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 74.0,
- "completions/max_terminated_length": 74.0,
- "completions/mean_length": 55.125,
- "completions/mean_terminated_length": 55.125,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "entropy": 1.0677781999111176,
- "epoch": 0.2926829268292683,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.14367543160915375,
- "kl": 1.2750254427373875e-05,
- "learning_rate": 7.567567567567568e-07,
- "loss": -0.001130133867263794,
- "num_tokens": 145294.0,
- "reward": 0.6871603727340698,
- "reward_std": 0.2714426517486572,
- "rewards/true_env_reward_fn/mean": 0.6871603727340698,
- "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 66.02083587646484,
+ "completions/mean_terminated_length": 66.02083587646484,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1611860394477844,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08851195871829987,
+ "kl": 1.2570341596074286e-05,
+ "learning_rate": 8.536585365853657e-07,
+ "loss": 0.021737128496170044,
+ "num_tokens": 883189.0,
+ "reward": 0.46058258414268494,
+ "reward_std": 0.2632383108139038,
+ "rewards/true_env_reward_fn/mean": 0.46058258414268494,
+ "rewards/true_env_reward_fn/std": 0.2632383108139038,
"step": 36,
- "step_time": 3.6285808550001093
+ "step_time": 8.370980583999994
},
{
"clip_ratio/high_max": 0.0,
@@ -1024,26 +1024,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 104.0,
- "completions/max_terminated_length": 104.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 49.0,
- "completions/min_terminated_length": 49.0,
- "entropy": 1.408882200717926,
- "epoch": 0.3008130081300813,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 75.58333587646484,
+ "completions/mean_terminated_length": 75.58333587646484,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.37085822224617,
+ "epoch": 0.9024390243902439,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.13866695761680603,
- "kl": 1.3317891898623202e-05,
- "learning_rate": 7.783783783783784e-07,
- "loss": 0.058712199330329895,
- "num_tokens": 148747.0,
- "reward": 0.638524055480957,
- "reward_std": 0.380489706993103,
- "rewards/true_env_reward_fn/mean": 0.638524055480957,
- "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "grad_norm": 0.05852028727531433,
+ "kl": 1.2957561011717189e-05,
+ "learning_rate": 8.780487804878048e-07,
+ "loss": -0.024281952530145645,
+ "num_tokens": 906801.0,
+ "reward": 0.5022324323654175,
+ "reward_std": 0.11637427657842636,
+ "rewards/true_env_reward_fn/mean": 0.5022324323654175,
+ "rewards/true_env_reward_fn/std": 0.11637428402900696,
"step": 37,
- "step_time": 4.57648780099953
+ "step_time": 10.285125336999727
},
{
"clip_ratio/high_max": 0.0,
@@ -1052,26 +1052,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 81.0,
- "completions/max_terminated_length": 81.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.3680316805839539,
- "epoch": 0.3089430894308943,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.13230997323989868,
- "kl": 1.1831724805233534e-05,
- "learning_rate": 8e-07,
- "loss": -0.06476183235645294,
- "num_tokens": 152794.0,
- "reward": 0.47908467054367065,
- "reward_std": 0.18681679666042328,
- "rewards/true_env_reward_fn/mean": 0.47908467054367065,
- "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "completions/max_length": 141.0,
+ "completions/max_terminated_length": 141.0,
+ "completions/mean_length": 65.14583587646484,
+ "completions/mean_terminated_length": 65.14583587646484,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.2760809361934662,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.09105321019887924,
+ "kl": 1.3129126955391257e-05,
+ "learning_rate": 9.024390243902439e-07,
+ "loss": -0.011838603764772415,
+ "num_tokens": 929536.0,
+ "reward": 0.49639374017715454,
+ "reward_std": 0.32166802883148193,
+ "rewards/true_env_reward_fn/mean": 0.49639371037483215,
+ "rewards/true_env_reward_fn/std": 0.32166802883148193,
"step": 38,
- "step_time": 3.627890882000429
+ "step_time": 12.449738128000035
},
{
"clip_ratio/high_max": 0.0,
@@ -1080,26 +1080,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 73.0,
- "completions/max_terminated_length": 73.0,
- "completions/mean_length": 56.875,
- "completions/mean_terminated_length": 56.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "entropy": 1.3124344944953918,
- "epoch": 0.3170731707317073,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20907950401306152,
- "kl": 1.4425510926230345e-05,
- "learning_rate": 7.999820918660971e-07,
- "loss": -0.014620006084442139,
- "num_tokens": 157337.0,
- "reward": 0.4882892966270447,
- "reward_std": 0.28137314319610596,
- "rewards/true_env_reward_fn/mean": 0.4882892966270447,
- "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "completions/max_length": 124.0,
+ "completions/max_terminated_length": 124.0,
+ "completions/mean_length": 72.08333587646484,
+ "completions/mean_terminated_length": 72.08333587646484,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2545586228370667,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06919296830892563,
+ "kl": 1.459557256566768e-05,
+ "learning_rate": 9.26829268292683e-07,
+ "loss": 0.021831180900335312,
+ "num_tokens": 950388.0,
+ "reward": 0.4879913330078125,
+ "reward_std": 0.24854585528373718,
+ "rewards/true_env_reward_fn/mean": 0.4879913330078125,
+ "rewards/true_env_reward_fn/std": 0.24854585528373718,
"step": 39,
- "step_time": 3.5362214279994078
+ "step_time": 10.279209028999958
},
{
"clip_ratio/high_max": 0.0,
@@ -1108,26 +1108,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 69.0,
- "completions/max_terminated_length": 69.0,
- "completions/mean_length": 51.625,
- "completions/mean_terminated_length": 51.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "entropy": 0.9928885996341705,
- "epoch": 0.3252032520325203,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 8.641438762424514e-05,
- "kl": 1.1731265658454504e-05,
- "learning_rate": 7.99928369067895e-07,
- "loss": 5.910313234380737e-07,
- "num_tokens": 160166.0,
- "reward": 0.6114685535430908,
- "reward_std": 0.1678776890039444,
- "rewards/true_env_reward_fn/mean": 0.6114685535430908,
- "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 74.20833587646484,
+ "completions/mean_terminated_length": 74.20833587646484,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2255937159061432,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06352153420448303,
+ "kl": 1.2041192348988261e-05,
+ "learning_rate": 9.512195121951218e-07,
+ "loss": -0.013997981324791908,
+ "num_tokens": 981254.0,
+ "reward": 0.39802420139312744,
+ "reward_std": 0.20212584733963013,
+ "rewards/true_env_reward_fn/mean": 0.39802420139312744,
+ "rewards/true_env_reward_fn/std": 0.20212584733963013,
"step": 40,
- "step_time": 3.1957039770022675
+ "step_time": 13.58010066599968
},
{
"clip_ratio/high_max": 0.0,
@@ -1136,26 +1136,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 54.75,
- "completions/mean_terminated_length": 54.75,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "entropy": 1.2997828722000122,
- "epoch": 0.3333333333333333,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.21271590888500214,
- "kl": 1.3209032658778597e-05,
- "learning_rate": 7.99838836415769e-07,
- "loss": 0.033298641443252563,
- "num_tokens": 165884.0,
- "reward": 0.2860966920852661,
- "reward_std": 0.2721884846687317,
- "rewards/true_env_reward_fn/mean": 0.2860966920852661,
- "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "completions/max_length": 139.0,
+ "completions/max_terminated_length": 139.0,
+ "completions/mean_length": 75.04167175292969,
+ "completions/mean_terminated_length": 75.04167175292969,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2703719735145569,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.6666666865348816,
+ "grad_norm": 0.045169439166784286,
+ "kl": 1.1270850109212915e-05,
+ "learning_rate": 9.756097560975609e-07,
+ "loss": -0.010194316506385803,
+ "num_tokens": 1009968.0,
+ "reward": 0.4517599940299988,
+ "reward_std": 0.11791092902421951,
+ "rewards/true_env_reward_fn/mean": 0.4517599642276764,
+ "rewards/true_env_reward_fn/std": 0.11791091412305832,
"step": 41,
- "step_time": 3.6851942720004445
+ "step_time": 10.35077203700007
},
{
"clip_ratio/high_max": 0.0,
@@ -1164,26 +1164,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 70.0,
- "completions/max_terminated_length": 70.0,
- "completions/mean_length": 48.875,
- "completions/mean_terminated_length": 48.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.4380556344985962,
- "epoch": 0.34146341463414637,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.17314757406711578,
- "kl": 9.354779194836738e-06,
- "learning_rate": 7.997135019265325e-07,
- "loss": 0.08398272097110748,
- "num_tokens": 172067.0,
- "reward": -0.003943998366594315,
- "reward_std": 0.13122709095478058,
- "rewards/true_env_reward_fn/mean": -0.003943998366594315,
- "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.33333587646484,
+ "completions/mean_terminated_length": 64.33333587646484,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.329576164484024,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08522730320692062,
+ "kl": 1.4469044799625408e-05,
+ "learning_rate": 1e-06,
+ "loss": -0.00014946190640330315,
+ "num_tokens": 1039032.0,
+ "reward": 0.33548423647880554,
+ "reward_std": 0.22271563112735748,
+ "rewards/true_env_reward_fn/mean": 0.33548423647880554,
+ "rewards/true_env_reward_fn/std": 0.22271563112735748,
"step": 42,
- "step_time": 3.545334507000007
+ "step_time": 10.548370664999993
},
{
"clip_ratio/high_max": 0.0,
@@ -1192,26 +1192,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 123.0,
- "completions/max_terminated_length": 123.0,
- "completions/mean_length": 71.0,
- "completions/mean_terminated_length": 71.0,
- "completions/min_length": 43.0,
- "completions/min_terminated_length": 43.0,
- "entropy": 1.163844645023346,
- "epoch": 0.34959349593495936,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15747681260108948,
- "kl": 1.3550960375141585e-05,
- "learning_rate": 7.995523768227198e-07,
- "loss": 0.05901219695806503,
- "num_tokens": 176427.0,
- "reward": 0.3297747075557709,
- "reward_std": 0.4647028148174286,
- "rewards/true_env_reward_fn/mean": 0.3297747075557709,
- "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "completions/max_length": 372.0,
+ "completions/max_terminated_length": 372.0,
+ "completions/mean_length": 70.02083587646484,
+ "completions/mean_terminated_length": 70.02083587646484,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.2357364892959595,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07030358910560608,
+ "kl": 1.3562755839302554e-05,
+ "learning_rate": 9.999818789066163e-07,
+ "loss": -0.02616041898727417,
+ "num_tokens": 1060833.0,
+ "reward": 0.5167371034622192,
+ "reward_std": 0.24280032515525818,
+ "rewards/true_env_reward_fn/mean": 0.5167370438575745,
+ "rewards/true_env_reward_fn/std": 0.24280032515525818,
"step": 43,
- "step_time": 5.4708715960005065
+ "step_time": 24.089396637999698
},
{
"clip_ratio/high_max": 0.0,
@@ -1220,26 +1220,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "entropy": 1.3323996663093567,
- "epoch": 0.35772357723577236,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.15227818489074707,
- "kl": 1.1237668786634458e-05,
- "learning_rate": 7.993554755315805e-07,
- "loss": 0.0660967156291008,
- "num_tokens": 181912.0,
- "reward": 0.22226500511169434,
- "reward_std": 0.2765512466430664,
- "rewards/true_env_reward_fn/mean": 0.22226500511169434,
- "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "completions/max_length": 234.0,
+ "completions/max_terminated_length": 234.0,
+ "completions/mean_length": 77.47917175292969,
+ "completions/mean_terminated_length": 77.47917175292969,
+ "completions/min_length": 14.0,
+ "completions/min_terminated_length": 14.0,
+ "entropy": 1.1693778038024902,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07017157226800919,
+ "kl": 1.332453393843025e-05,
+ "learning_rate": 9.999275169399612e-07,
+ "loss": -0.006466507911682129,
+ "num_tokens": 1088648.0,
+ "reward": 0.4498252272605896,
+ "reward_std": 0.21398545801639557,
+ "rewards/true_env_reward_fn/mean": 0.4498251974582672,
+ "rewards/true_env_reward_fn/std": 0.21398545801639557,
"step": 44,
- "step_time": 3.940563359999942
+ "step_time": 19.39071501599983
},
{
"clip_ratio/high_max": 0.0,
@@ -1248,26 +1248,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 56.375,
- "completions/mean_terminated_length": 56.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.241302490234375,
- "epoch": 0.36585365853658536,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1711702048778534,
- "kl": 1.1479866316221887e-05,
- "learning_rate": 7.991228156837879e-07,
- "loss": 0.0959811806678772,
- "num_tokens": 186099.0,
- "reward": 0.4569639563560486,
- "reward_std": 0.356449693441391,
- "rewards/true_env_reward_fn/mean": 0.4569639563560486,
- "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 72.16667175292969,
+ "completions/mean_terminated_length": 72.16667175292969,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3268415927886963,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.06632921099662781,
+ "kl": 1.4458733630817733e-05,
+ "learning_rate": 9.99836918040428e-07,
+ "loss": -0.03534461930394173,
+ "num_tokens": 1117096.0,
+ "reward": 0.4053138196468353,
+ "reward_std": 0.21476909518241882,
+ "rewards/true_env_reward_fn/mean": 0.4053138196468353,
+ "rewards/true_env_reward_fn/std": 0.21476909518241882,
"step": 45,
- "step_time": 3.947248132999448
+ "step_time": 13.893569495999827
},
{
"clip_ratio/high_max": 0.0,
@@ -1276,26 +1276,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 83.0,
- "completions/max_terminated_length": 83.0,
- "completions/mean_length": 66.625,
- "completions/mean_terminated_length": 66.625,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "entropy": 1.5153677463531494,
- "epoch": 0.37398373983739835,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.22329360246658325,
- "kl": 1.3615897842100821e-05,
- "learning_rate": 7.988544181118608e-07,
- "loss": 0.07407481223344803,
- "num_tokens": 192056.0,
- "reward": 0.2950569987297058,
- "reward_std": 0.2872281074523926,
- "rewards/true_env_reward_fn/mean": 0.2950569987297058,
- "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "completions/max_length": 160.0,
+ "completions/max_terminated_length": 160.0,
+ "completions/mean_length": 70.16667175292969,
+ "completions/mean_terminated_length": 70.16667175292969,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2670875787734985,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.08321154117584229,
+ "kl": 1.4837954950053245e-05,
+ "learning_rate": 9.997100887750215e-07,
+ "loss": -0.039235007017850876,
+ "num_tokens": 1136480.0,
+ "reward": 0.48141974210739136,
+ "reward_std": 0.2837103307247162,
+ "rewards/true_env_reward_fn/mean": 0.48141971230506897,
+ "rewards/true_env_reward_fn/std": 0.2837103009223938,
"step": 46,
- "step_time": 4.1211709569997765
+ "step_time": 10.50698806499986
},
{
"clip_ratio/high_max": 0.0,
@@ -1304,26 +1304,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 62.0,
- "completions/max_terminated_length": 62.0,
- "completions/mean_length": 54.5,
- "completions/mean_terminated_length": 54.5,
- "completions/min_length": 44.0,
- "completions/min_terminated_length": 44.0,
- "entropy": 1.105223298072815,
- "epoch": 0.3821138211382114,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.1306377500295639,
- "kl": 1.2826244528696407e-05,
- "learning_rate": 7.985503068482974e-07,
- "loss": 0.014609627425670624,
- "num_tokens": 195544.0,
- "reward": 0.5289265513420105,
- "reward_std": 0.3883950710296631,
- "rewards/true_env_reward_fn/mean": 0.5289265513420105,
- "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "completions/max_length": 159.0,
+ "completions/max_terminated_length": 159.0,
+ "completions/mean_length": 76.1875,
+ "completions/mean_terminated_length": 76.1875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3037313222885132,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.061912886798381805,
+ "kl": 1.283655774386716e-05,
+ "learning_rate": 9.995470383368808e-07,
+ "loss": -0.01992109790444374,
+ "num_tokens": 1162249.0,
+ "reward": 0.49922606348991394,
+ "reward_std": 0.2621309757232666,
+ "rewards/true_env_reward_fn/mean": 0.49922606348991394,
+ "rewards/true_env_reward_fn/std": 0.2621309757232666,
"step": 47,
- "step_time": 2.938600743000279
+ "step_time": 12.964419044000124
},
{
"clip_ratio/high_max": 0.0,
@@ -1332,26 +1332,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.375,
- "completions/mean_terminated_length": 67.375,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "entropy": 1.5243317484855652,
- "epoch": 0.3902439024390244,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.20161111652851105,
- "kl": 1.4497059055429418e-05,
- "learning_rate": 7.982105091234235e-07,
- "loss": 0.23342597484588623,
- "num_tokens": 198691.0,
- "reward": 0.45001715421676636,
- "reward_std": 0.2565726041793823,
- "rewards/true_env_reward_fn/mean": 0.45001715421676636,
- "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "completions/max_length": 231.0,
+ "completions/max_terminated_length": 231.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2007178366184235,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.1666666716337204,
+ "grad_norm": 0.0889662653207779,
+ "kl": 1.6228528693318367e-05,
+ "learning_rate": 9.993477785446149e-07,
+ "loss": 0.045945264399051666,
+ "num_tokens": 1184555.0,
+ "reward": 0.42501482367515564,
+ "reward_std": 0.27350595593452454,
+ "rewards/true_env_reward_fn/mean": 0.42501482367515564,
+ "rewards/true_env_reward_fn/std": 0.27350592613220215,
"step": 48,
- "step_time": 4.91795033499875
+ "step_time": 17.23041258299986
},
{
"clip_ratio/high_max": 0.0,
@@ -1360,26 +1360,26 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 79.0,
- "completions/max_terminated_length": 79.0,
- "completions/mean_length": 55.375,
- "completions/mean_terminated_length": 55.375,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.2544435858726501,
- "epoch": 0.3983739837398374,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.11793916672468185,
- "kl": 1.3676196886081016e-05,
- "learning_rate": 7.978350553629554e-07,
- "loss": -0.016418367624282837,
- "num_tokens": 202994.0,
- "reward": 0.4054500162601471,
- "reward_std": 0.20634961128234863,
- "rewards/true_env_reward_fn/mean": 0.4054500162601471,
- "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.9375,
+ "completions/mean_terminated_length": 55.9375,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.182040810585022,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.08547856658697128,
+ "kl": 1.571832831359643e-05,
+ "learning_rate": 9.991123238414453e-07,
+ "loss": 0.02548346482217312,
+ "num_tokens": 1208384.0,
+ "reward": 0.3845663070678711,
+ "reward_std": 0.315467894077301,
+ "rewards/true_env_reward_fn/mean": 0.3845663070678711,
+ "rewards/true_env_reward_fn/std": 0.31546786427497864,
"step": 49,
- "step_time": 3.626596234000317
+ "step_time": 8.691208415999881
},
{
"clip_ratio/high_max": 0.0,
@@ -1388,32 +1388,32 @@
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 62.5,
- "completions/mean_terminated_length": 62.5,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "entropy": 1.022342562675476,
- "epoch": 0.4065040650406504,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.16596083343029022,
- "kl": 1.1194244052603608e-05,
- "learning_rate": 7.974239791852739e-07,
- "loss": 0.0499756895005703,
- "num_tokens": 205770.0,
- "reward": 0.5639185309410095,
- "reward_std": 0.1721728891134262,
- "rewards/true_env_reward_fn/mean": 0.5639185309410095,
- "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2306177020072937,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 0.3333333432674408,
+ "grad_norm": 0.07395736873149872,
+ "kl": 1.2643881973417592e-05,
+ "learning_rate": 9.988406912941589e-07,
+ "loss": -0.04186868295073509,
+ "num_tokens": 1227700.0,
+ "reward": 0.5068289637565613,
+ "reward_std": 0.31324177980422974,
+ "rewards/true_env_reward_fn/mean": 0.5068289637565613,
+ "rewards/true_env_reward_fn/std": 0.31324175000190735,
"step": 50,
- "step_time": 3.9679293660010444
+ "step_time": 10.162109979000206
}
],
"logging_steps": 1,
- "max_steps": 369,
- "num_input_tokens_seen": 205770,
- "num_train_epochs": 3,
+ "max_steps": 410,
+ "num_input_tokens_seen": 1227700,
+ "num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -1428,7 +1428,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 4,
+ "train_batch_size": 12,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-50/training_args.bin b/checkpoint-50/training_args.bin
index 8d94c3c38f17faf8a60976b504514708acad4864..2276397b5f715b1fed46fa8d458d3bde360f7b81 100644
--- a/checkpoint-50/training_args.bin
+++ b/checkpoint-50/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+oid sha256:8c078149c5f6d3dea09968db8e43b83e4250f9a28eaa91663c54a1a1964152d2
size 6776
diff --git a/ref/adapter_model.safetensors b/ref/adapter_model.safetensors
index 1748ecc34d0d4aae1e8bc8135cb16bc901705fd4..4b516b95e2bde01b4a51b7977bce639f00946144 100644
--- a/ref/adapter_model.safetensors
+++ b/ref/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+oid sha256:c5668a13f5c891568fbb8579d9c51e0cc04a2346765ac750be5c49316caeb7d9
size 8731128
diff --git a/training_metadata.json b/training_metadata.json
index e58bec5fe7747648f889ae244ee4fdd65dad28fc..c907501cc0c11907af6360a16292cff97bf81349 100644
--- a/training_metadata.json
+++ b/training_metadata.json
@@ -8,15 +8,15 @@
2,
3
],
- "num_train_epochs": 3.0,
- "per_device_batch": 4,
- "grad_accum": 2,
- "num_generations": 4,
- "lr": 8e-07,
+ "num_train_epochs": 10.0,
+ "per_device_batch": 12,
+ "grad_accum": 4,
+ "num_generations": 8,
+ "lr": 1e-06,
"beta": 0.05,
"gamma": 0.98,
"seed": 42,
"n_unique_prompts": 247,
- "n_state_snapshots": 682,
+ "n_state_snapshots": 612,
"use_vllm": false
}
\ No newline at end of file