Upload generator checkpoints for round 001
Browse files- self_play_hf_a10g_train/round_001/generator_train/README.md +1 -1
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/model.safetensors +1 -1
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/optimizer.pt +1 -1
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/trainer_state.json +389 -389
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/model.safetensors +1 -1
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/optimizer.pt +1 -1
- self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/trainer_state.json +486 -486
- self_play_hf_a10g_train/round_001/generator_train/final_model/model.safetensors +1 -1
self_play_hf_a10g_train/round_001/generator_train/README.md
CHANGED
|
@@ -27,7 +27,7 @@ print(output["generated_text"])
|
|
| 27 |
|
| 28 |
## Training procedure
|
| 29 |
|
| 30 |
-
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/siddeshwar2004-international-institute-of-information-te/osint-self-play-train/runs/
|
| 31 |
|
| 32 |
|
| 33 |
|
|
|
|
| 27 |
|
| 28 |
## Training procedure
|
| 29 |
|
| 30 |
+
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/siddeshwar2004-international-institute-of-information-te/osint-self-play-train/runs/d6lveb1e)
|
| 31 |
|
| 32 |
|
| 33 |
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1976163472
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0827989c90bcaad483dee84b62c1ba69fdf377e659087667ae2a28e1992a2fc6
|
| 3 |
size 1976163472
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/optimizer.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 3952509771
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ceec2f7113291001d1654c27484e896c530e9cfd6710e3ffcfdcc4fb0eeee677
|
| 3 |
size 3952509771
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-40/trainer_state.json
CHANGED
|
@@ -25,7 +25,7 @@
|
|
| 25 |
"entropy": 1.9362258911132812,
|
| 26 |
"epoch": 0.041666666666666664,
|
| 27 |
"frac_reward_zero_std": 0.0,
|
| 28 |
-
"grad_norm": 3.
|
| 29 |
"kl": 0.0005498419050127268,
|
| 30 |
"learning_rate": 5e-06,
|
| 31 |
"loss": 0.13995857536792755,
|
|
@@ -35,59 +35,59 @@
|
|
| 35 |
"rewards/GeneratorRewardFunction/mean": -0.4352343678474426,
|
| 36 |
"rewards/GeneratorRewardFunction/std": 0.306624174118042,
|
| 37 |
"step": 1,
|
| 38 |
-
"step_time": 12.
|
| 39 |
},
|
| 40 |
{
|
| 41 |
-
"clip_ratio/high_max": 0.
|
| 42 |
-
"clip_ratio/high_mean": 0.
|
| 43 |
-
"clip_ratio/low_mean": 0.
|
| 44 |
-
"clip_ratio/low_min": 0.
|
| 45 |
-
"clip_ratio/region_mean": 0.
|
| 46 |
-
"entropy": 1.
|
| 47 |
"epoch": 0.08333333333333333,
|
| 48 |
-
"grad_norm": 2.
|
| 49 |
-
"kl": 0.
|
| 50 |
"learning_rate": 4.9000000000000005e-06,
|
| 51 |
-
"loss": -0.
|
| 52 |
"step": 2,
|
| 53 |
-
"step_time": 0.
|
| 54 |
},
|
| 55 |
{
|
| 56 |
-
"clip_ratio/high_max": 0.
|
| 57 |
-
"clip_ratio/high_mean": 0.
|
| 58 |
-
"clip_ratio/low_mean": 0.
|
| 59 |
-
"clip_ratio/low_min": 0.
|
| 60 |
"clip_ratio/region_mean": 0.02799479104578495,
|
| 61 |
-
"entropy": 1.
|
| 62 |
"epoch": 0.125,
|
| 63 |
-
"grad_norm": 2.
|
| 64 |
-
"kl": 0.
|
| 65 |
"learning_rate": 4.800000000000001e-06,
|
| 66 |
-
"loss": 0.
|
| 67 |
"step": 3,
|
| 68 |
-
"step_time": 0.
|
| 69 |
},
|
| 70 |
{
|
| 71 |
-
"clip_ratio/high_max": 0.
|
| 72 |
-
"clip_ratio/high_mean": 0.
|
| 73 |
"clip_ratio/low_mean": 0.010416666977107525,
|
| 74 |
"clip_ratio/low_min": 0.010416666977107525,
|
| 75 |
-
"clip_ratio/region_mean": 0.
|
| 76 |
-
"entropy": 1.
|
| 77 |
"epoch": 0.16666666666666666,
|
| 78 |
-
"grad_norm": 1.
|
| 79 |
-
"kl": 0.
|
| 80 |
"learning_rate": 4.7e-06,
|
| 81 |
-
"loss": -0.
|
| 82 |
"step": 4,
|
| 83 |
-
"step_time": 0.
|
| 84 |
},
|
| 85 |
{
|
| 86 |
-
"clip_ratio/high_max": 0.
|
| 87 |
-
"clip_ratio/high_mean": 0.
|
| 88 |
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 89 |
"clip_ratio/low_min": 0.0006510416860692203,
|
| 90 |
-
"clip_ratio/region_mean": 0.
|
| 91 |
"completions/clipped_ratio": 1.0,
|
| 92 |
"completions/max_length": 384.0,
|
| 93 |
"completions/max_terminated_length": 0.0,
|
|
@@ -95,65 +95,65 @@
|
|
| 95 |
"completions/mean_terminated_length": 0.0,
|
| 96 |
"completions/min_length": 384.0,
|
| 97 |
"completions/min_terminated_length": 0.0,
|
| 98 |
-
"entropy": 1.
|
| 99 |
"epoch": 0.20833333333333334,
|
| 100 |
"frac_reward_zero_std": 0.0,
|
| 101 |
-
"grad_norm":
|
| 102 |
-
"kl": 0.
|
| 103 |
"learning_rate": 4.600000000000001e-06,
|
| 104 |
-
"loss": 0.
|
| 105 |
"num_tokens": 50440.0,
|
| 106 |
-
"reward": -0.
|
| 107 |
-
"reward_std": 0.
|
| 108 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 109 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 110 |
"step": 5,
|
| 111 |
-
"step_time": 12.
|
| 112 |
},
|
| 113 |
{
|
| 114 |
-
"clip_ratio/high_max": 0.
|
| 115 |
-
"clip_ratio/high_mean": 0.
|
| 116 |
-
"clip_ratio/low_mean": 0.
|
| 117 |
-
"clip_ratio/low_min": 0.
|
| 118 |
-
"clip_ratio/region_mean": 0.
|
| 119 |
-
"entropy": 1.
|
| 120 |
"epoch": 0.25,
|
| 121 |
-
"grad_norm": 2.
|
| 122 |
-
"kl": 0.
|
| 123 |
"learning_rate": 4.5e-06,
|
| 124 |
-
"loss":
|
| 125 |
"step": 6,
|
| 126 |
-
"step_time": 0.
|
| 127 |
},
|
| 128 |
{
|
| 129 |
-
"clip_ratio/high_max": 0.
|
| 130 |
-
"clip_ratio/high_mean": 0.
|
| 131 |
-
"clip_ratio/low_mean": 0.
|
| 132 |
-
"clip_ratio/low_min": 0.
|
| 133 |
-
"clip_ratio/region_mean": 0.
|
| 134 |
-
"entropy": 1.
|
| 135 |
"epoch": 0.2916666666666667,
|
| 136 |
-
"grad_norm":
|
| 137 |
-
"kl": 0.
|
| 138 |
"learning_rate": 4.4e-06,
|
| 139 |
-
"loss": 0.
|
| 140 |
"step": 7,
|
| 141 |
-
"step_time": 0.
|
| 142 |
},
|
| 143 |
{
|
| 144 |
"clip_ratio/high_max": 0.029296875,
|
| 145 |
"clip_ratio/high_mean": 0.029296875,
|
| 146 |
-
"clip_ratio/low_mean": 0.
|
| 147 |
-
"clip_ratio/low_min": 0.
|
| 148 |
-
"clip_ratio/region_mean": 0.
|
| 149 |
-
"entropy":
|
| 150 |
"epoch": 0.3333333333333333,
|
| 151 |
-
"grad_norm":
|
| 152 |
-
"kl": 0.
|
| 153 |
"learning_rate": 4.3e-06,
|
| 154 |
-
"loss":
|
| 155 |
"step": 8,
|
| 156 |
-
"step_time": 0.
|
| 157 |
},
|
| 158 |
{
|
| 159 |
"clip_ratio/high_max": 0.0,
|
|
@@ -168,72 +168,72 @@
|
|
| 168 |
"completions/mean_terminated_length": 0.0,
|
| 169 |
"completions/min_length": 384.0,
|
| 170 |
"completions/min_terminated_length": 0.0,
|
| 171 |
-
"entropy":
|
| 172 |
"epoch": 0.375,
|
| 173 |
"frac_reward_zero_std": 0.25,
|
| 174 |
-
"grad_norm":
|
| 175 |
-
"kl": 0.
|
| 176 |
"learning_rate": 4.2000000000000004e-06,
|
| 177 |
-
"loss": -0.
|
| 178 |
"num_tokens": 75884.0,
|
| 179 |
-
"reward": -0.
|
| 180 |
-
"reward_std": 0.
|
| 181 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 182 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 183 |
"step": 9,
|
| 184 |
-
"step_time":
|
| 185 |
},
|
| 186 |
{
|
| 187 |
-
"clip_ratio/high_max": 0.
|
| 188 |
-
"clip_ratio/high_mean": 0.
|
| 189 |
-
"clip_ratio/low_mean": 0.
|
| 190 |
-
"clip_ratio/low_min": 0.
|
| 191 |
-
"clip_ratio/region_mean": 0.
|
| 192 |
-
"entropy": 1.
|
| 193 |
"epoch": 0.4166666666666667,
|
| 194 |
-
"grad_norm": 2.
|
| 195 |
-
"kl": 0.
|
| 196 |
"learning_rate": 4.1e-06,
|
| 197 |
-
"loss": 0.
|
| 198 |
"step": 10,
|
| 199 |
-
"step_time": 0.
|
| 200 |
},
|
| 201 |
{
|
| 202 |
-
"clip_ratio/high_max": 0.
|
| 203 |
-
"clip_ratio/high_mean": 0.
|
| 204 |
-
"clip_ratio/low_mean": 0.
|
| 205 |
-
"clip_ratio/low_min": 0.
|
| 206 |
-
"clip_ratio/region_mean": 0.
|
| 207 |
-
"entropy":
|
| 208 |
"epoch": 0.4583333333333333,
|
| 209 |
-
"grad_norm": 2.
|
| 210 |
-
"kl": 0.
|
| 211 |
"learning_rate": 4.000000000000001e-06,
|
| 212 |
-
"loss": 0.
|
| 213 |
"step": 11,
|
| 214 |
-
"step_time": 0.
|
| 215 |
},
|
| 216 |
{
|
| 217 |
-
"clip_ratio/high_max": 0.
|
| 218 |
-
"clip_ratio/high_mean": 0.
|
| 219 |
-
"clip_ratio/low_mean": 0.
|
| 220 |
-
"clip_ratio/low_min": 0.
|
| 221 |
-
"clip_ratio/region_mean": 0.
|
| 222 |
-
"entropy": 1.
|
| 223 |
"epoch": 0.5,
|
| 224 |
-
"grad_norm":
|
| 225 |
-
"kl": 0.
|
| 226 |
"learning_rate": 3.900000000000001e-06,
|
| 227 |
-
"loss":
|
| 228 |
"step": 12,
|
| 229 |
-
"step_time": 0.
|
| 230 |
},
|
| 231 |
{
|
| 232 |
-
"clip_ratio/high_max": 0.
|
| 233 |
-
"clip_ratio/high_mean": 0.
|
| 234 |
-
"clip_ratio/low_mean": 0.
|
| 235 |
-
"clip_ratio/low_min": 0.
|
| 236 |
-
"clip_ratio/region_mean": 0.
|
| 237 |
"completions/clipped_ratio": 1.0,
|
| 238 |
"completions/max_length": 384.0,
|
| 239 |
"completions/max_terminated_length": 0.0,
|
|
@@ -241,65 +241,65 @@
|
|
| 241 |
"completions/mean_terminated_length": 0.0,
|
| 242 |
"completions/min_length": 384.0,
|
| 243 |
"completions/min_terminated_length": 0.0,
|
| 244 |
-
"entropy": 1.
|
| 245 |
"epoch": 0.5416666666666666,
|
| 246 |
-
"frac_reward_zero_std": 0.
|
| 247 |
-
"grad_norm": 2.
|
| 248 |
-
"kl": 0.
|
| 249 |
"learning_rate": 3.8000000000000005e-06,
|
| 250 |
-
"loss": 0.
|
| 251 |
"num_tokens": 101112.0,
|
| 252 |
-
"reward": -0.
|
| 253 |
-
"reward_std": 0.
|
| 254 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 255 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 256 |
"step": 13,
|
| 257 |
-
"step_time": 12.
|
| 258 |
},
|
| 259 |
{
|
| 260 |
-
"clip_ratio/high_max": 0.
|
| 261 |
-
"clip_ratio/high_mean": 0.
|
| 262 |
-
"clip_ratio/low_mean": 0.
|
| 263 |
-
"clip_ratio/low_min": 0.
|
| 264 |
-
"clip_ratio/region_mean": 0.
|
| 265 |
-
"entropy": 1.
|
| 266 |
"epoch": 0.5833333333333334,
|
| 267 |
-
"grad_norm":
|
| 268 |
-
"kl": 0.
|
| 269 |
"learning_rate": 3.7e-06,
|
| 270 |
-
"loss": 0.
|
| 271 |
"step": 14,
|
| 272 |
-
"step_time": 0.
|
| 273 |
},
|
| 274 |
{
|
| 275 |
-
"clip_ratio/high_max": 0.
|
| 276 |
-
"clip_ratio/high_mean": 0.
|
| 277 |
-
"clip_ratio/low_mean": 0.
|
| 278 |
-
"clip_ratio/low_min": 0.
|
| 279 |
-
"clip_ratio/region_mean": 0.
|
| 280 |
-
"entropy": 1.
|
| 281 |
"epoch": 0.625,
|
| 282 |
-
"grad_norm":
|
| 283 |
-
"kl": 0.
|
| 284 |
"learning_rate": 3.6000000000000003e-06,
|
| 285 |
-
"loss":
|
| 286 |
"step": 15,
|
| 287 |
-
"step_time": 0.
|
| 288 |
},
|
| 289 |
{
|
| 290 |
-
"clip_ratio/high_max": 0.
|
| 291 |
-
"clip_ratio/high_mean": 0.
|
| 292 |
-
"clip_ratio/low_mean": 0.
|
| 293 |
-
"clip_ratio/low_min": 0.
|
| 294 |
-
"clip_ratio/region_mean": 0.
|
| 295 |
-
"entropy":
|
| 296 |
"epoch": 0.6666666666666666,
|
| 297 |
-
"grad_norm":
|
| 298 |
-
"kl": 0.
|
| 299 |
"learning_rate": 3.5e-06,
|
| 300 |
-
"loss": -0.
|
| 301 |
"step": 16,
|
| 302 |
-
"step_time": 0.
|
| 303 |
},
|
| 304 |
{
|
| 305 |
"clip_ratio/high_max": 0.0,
|
|
@@ -314,72 +314,72 @@
|
|
| 314 |
"completions/mean_terminated_length": 0.0,
|
| 315 |
"completions/min_length": 384.0,
|
| 316 |
"completions/min_terminated_length": 0.0,
|
| 317 |
-
"entropy":
|
| 318 |
"epoch": 0.7083333333333334,
|
| 319 |
"frac_reward_zero_std": 0.0,
|
| 320 |
-
"grad_norm":
|
| 321 |
-
"kl": 0.
|
| 322 |
"learning_rate": 3.4000000000000005e-06,
|
| 323 |
-
"loss": 0.
|
| 324 |
"num_tokens": 126436.0,
|
| 325 |
-
"reward": -0.
|
| 326 |
-
"reward_std": 0.
|
| 327 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 328 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 329 |
"step": 17,
|
| 330 |
-
"step_time": 12.
|
| 331 |
},
|
| 332 |
{
|
| 333 |
-
"clip_ratio/high_max": 0.
|
| 334 |
-
"clip_ratio/high_mean": 0.
|
| 335 |
-
"clip_ratio/low_mean": 0.
|
| 336 |
-
"clip_ratio/low_min": 0.
|
| 337 |
-
"clip_ratio/region_mean": 0.
|
| 338 |
-
"entropy":
|
| 339 |
"epoch": 0.75,
|
| 340 |
-
"grad_norm":
|
| 341 |
-
"kl": 0.
|
| 342 |
"learning_rate": 3.3000000000000006e-06,
|
| 343 |
-
"loss": -0.
|
| 344 |
"step": 18,
|
| 345 |
-
"step_time": 0.
|
| 346 |
},
|
| 347 |
{
|
| 348 |
-
"clip_ratio/high_max": 0.
|
| 349 |
-
"clip_ratio/high_mean": 0.
|
| 350 |
-
"clip_ratio/low_mean": 0.
|
| 351 |
-
"clip_ratio/low_min": 0.
|
| 352 |
-
"clip_ratio/region_mean": 0.
|
| 353 |
-
"entropy":
|
| 354 |
"epoch": 0.7916666666666666,
|
| 355 |
-
"grad_norm":
|
| 356 |
-
"kl": 0.
|
| 357 |
"learning_rate": 3.2000000000000003e-06,
|
| 358 |
-
"loss":
|
| 359 |
"step": 19,
|
| 360 |
-
"step_time": 0.
|
| 361 |
},
|
| 362 |
{
|
| 363 |
-
"clip_ratio/high_max": 0.
|
| 364 |
-
"clip_ratio/high_mean": 0.
|
| 365 |
-
"clip_ratio/low_mean": 0.
|
| 366 |
-
"clip_ratio/low_min": 0.
|
| 367 |
-
"clip_ratio/region_mean": 0.
|
| 368 |
-
"entropy": 1.
|
| 369 |
"epoch": 0.8333333333333334,
|
| 370 |
-
"grad_norm":
|
| 371 |
-
"kl": 0.
|
| 372 |
"learning_rate": 3.1000000000000004e-06,
|
| 373 |
-
"loss": 0.
|
| 374 |
"step": 20,
|
| 375 |
-
"step_time": 0.
|
| 376 |
},
|
| 377 |
{
|
| 378 |
-
"clip_ratio/high_max": 0.
|
| 379 |
-
"clip_ratio/high_mean": 0.
|
| 380 |
"clip_ratio/low_mean": 0.0,
|
| 381 |
"clip_ratio/low_min": 0.0,
|
| 382 |
-
"clip_ratio/region_mean": 0.
|
| 383 |
"completions/clipped_ratio": 1.0,
|
| 384 |
"completions/max_length": 384.0,
|
| 385 |
"completions/max_terminated_length": 0.0,
|
|
@@ -387,72 +387,72 @@
|
|
| 387 |
"completions/mean_terminated_length": 0.0,
|
| 388 |
"completions/min_length": 384.0,
|
| 389 |
"completions/min_terminated_length": 0.0,
|
| 390 |
-
"entropy": 1.
|
| 391 |
"epoch": 0.875,
|
| 392 |
-
"frac_reward_zero_std": 0.
|
| 393 |
-
"grad_norm": 2.
|
| 394 |
-
"kl": 0.
|
| 395 |
"learning_rate": 3e-06,
|
| 396 |
-
"loss": -0.
|
| 397 |
"num_tokens": 151596.0,
|
| 398 |
-
"reward": -0.
|
| 399 |
-
"reward_std": 0.
|
| 400 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 401 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 402 |
"step": 21,
|
| 403 |
-
"step_time":
|
| 404 |
},
|
| 405 |
{
|
| 406 |
-
"clip_ratio/high_max": 0.
|
| 407 |
-
"clip_ratio/high_mean": 0.
|
| 408 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 409 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 410 |
-
"clip_ratio/region_mean": 0.
|
| 411 |
-
"entropy": 1.
|
| 412 |
"epoch": 0.9166666666666666,
|
| 413 |
-
"grad_norm":
|
| 414 |
-
"kl": 0.
|
| 415 |
"learning_rate": 2.9e-06,
|
| 416 |
-
"loss":
|
| 417 |
"step": 22,
|
| 418 |
-
"step_time": 0.
|
| 419 |
},
|
| 420 |
{
|
| 421 |
-
"clip_ratio/high_max": 0.
|
| 422 |
-
"clip_ratio/high_mean": 0.
|
| 423 |
-
"clip_ratio/low_mean": 0.
|
| 424 |
-
"clip_ratio/low_min": 0.
|
| 425 |
-
"clip_ratio/region_mean": 0.
|
| 426 |
-
"entropy": 1.
|
| 427 |
"epoch": 0.9583333333333334,
|
| 428 |
-
"grad_norm": 2.
|
| 429 |
-
"kl": 0.
|
| 430 |
"learning_rate": 2.8000000000000003e-06,
|
| 431 |
-
"loss": 0.
|
| 432 |
"step": 23,
|
| 433 |
-
"step_time": 0.
|
| 434 |
},
|
| 435 |
{
|
| 436 |
-
"clip_ratio/high_max": 0.
|
| 437 |
-
"clip_ratio/high_mean": 0.
|
| 438 |
-
"clip_ratio/low_mean": 0.
|
| 439 |
-
"clip_ratio/low_min": 0.
|
| 440 |
-
"clip_ratio/region_mean": 0.
|
| 441 |
-
"entropy": 0.
|
| 442 |
"epoch": 1.0,
|
| 443 |
-
"grad_norm": 1.
|
| 444 |
-
"kl": 0.
|
| 445 |
"learning_rate": 2.7000000000000004e-06,
|
| 446 |
-
"loss": 0.
|
| 447 |
"step": 24,
|
| 448 |
-
"step_time": 0.
|
| 449 |
},
|
| 450 |
{
|
| 451 |
-
"clip_ratio/high_max": 0.
|
| 452 |
-
"clip_ratio/high_mean": 0.
|
| 453 |
"clip_ratio/low_mean": 0.0,
|
| 454 |
"clip_ratio/low_min": 0.0,
|
| 455 |
-
"clip_ratio/region_mean": 0.
|
| 456 |
"completions/clipped_ratio": 1.0,
|
| 457 |
"completions/max_length": 384.0,
|
| 458 |
"completions/max_terminated_length": 0.0,
|
|
@@ -460,20 +460,20 @@
|
|
| 460 |
"completions/mean_terminated_length": 0.0,
|
| 461 |
"completions/min_length": 384.0,
|
| 462 |
"completions/min_terminated_length": 0.0,
|
| 463 |
-
"entropy": 1.
|
| 464 |
"epoch": 1.0416666666666667,
|
| 465 |
"frac_reward_zero_std": 0.0,
|
| 466 |
-
"grad_norm": 2.
|
| 467 |
-
"kl": 0.
|
| 468 |
"learning_rate": 2.6e-06,
|
| 469 |
-
"loss": 0.
|
| 470 |
"num_tokens": 177212.0,
|
| 471 |
-
"reward": -0.
|
| 472 |
-
"reward_std": 0.
|
| 473 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 474 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 475 |
"step": 25,
|
| 476 |
-
"step_time": 12.
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"clip_ratio/high_max": 0.00390625,
|
|
@@ -481,51 +481,51 @@
|
|
| 481 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 482 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 483 |
"clip_ratio/region_mean": 0.0052083334885537624,
|
| 484 |
-
"entropy":
|
| 485 |
"epoch": 1.0833333333333333,
|
| 486 |
-
"grad_norm":
|
| 487 |
-
"kl": 0.
|
| 488 |
"learning_rate": 2.5e-06,
|
| 489 |
-
"loss": -0.
|
| 490 |
"step": 26,
|
| 491 |
-
"step_time": 0.
|
| 492 |
},
|
| 493 |
{
|
| 494 |
-
"clip_ratio/high_max": 0.
|
| 495 |
-
"clip_ratio/high_mean": 0.
|
| 496 |
-
"clip_ratio/low_mean": 0.
|
| 497 |
-
"clip_ratio/low_min": 0.
|
| 498 |
-
"clip_ratio/region_mean": 0.
|
| 499 |
-
"entropy": 1.
|
| 500 |
"epoch": 1.125,
|
| 501 |
-
"grad_norm":
|
| 502 |
-
"kl": 0.
|
| 503 |
"learning_rate": 2.4000000000000003e-06,
|
| 504 |
-
"loss": -0.
|
| 505 |
"step": 27,
|
| 506 |
-
"step_time": 0.
|
| 507 |
},
|
| 508 |
{
|
| 509 |
-
"clip_ratio/high_max": 0.
|
| 510 |
-
"clip_ratio/high_mean": 0.
|
| 511 |
-
"clip_ratio/low_mean": 0.
|
| 512 |
-
"clip_ratio/low_min": 0.
|
| 513 |
-
"clip_ratio/region_mean": 0.
|
| 514 |
-
"entropy":
|
| 515 |
"epoch": 1.1666666666666667,
|
| 516 |
-
"grad_norm":
|
| 517 |
-
"kl": 0.
|
| 518 |
"learning_rate": 2.3000000000000004e-06,
|
| 519 |
-
"loss": 0.
|
| 520 |
"step": 28,
|
| 521 |
-
"step_time": 0.
|
| 522 |
},
|
| 523 |
{
|
| 524 |
-
"clip_ratio/high_max": 0.
|
| 525 |
-
"clip_ratio/high_mean": 0.
|
| 526 |
"clip_ratio/low_mean": 0.0,
|
| 527 |
"clip_ratio/low_min": 0.0,
|
| 528 |
-
"clip_ratio/region_mean": 0.
|
| 529 |
"completions/clipped_ratio": 1.0,
|
| 530 |
"completions/max_length": 384.0,
|
| 531 |
"completions/max_terminated_length": 0.0,
|
|
@@ -533,65 +533,65 @@
|
|
| 533 |
"completions/mean_terminated_length": 0.0,
|
| 534 |
"completions/min_length": 384.0,
|
| 535 |
"completions/min_terminated_length": 0.0,
|
| 536 |
-
"entropy":
|
| 537 |
"epoch": 1.2083333333333333,
|
| 538 |
"frac_reward_zero_std": 0.0,
|
| 539 |
-
"grad_norm":
|
| 540 |
-
"kl": 0.
|
| 541 |
"learning_rate": 2.2e-06,
|
| 542 |
-
"loss":
|
| 543 |
"num_tokens": 202200.0,
|
| 544 |
-
"reward": -0.
|
| 545 |
-
"reward_std": 0.
|
| 546 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 547 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 548 |
"step": 29,
|
| 549 |
-
"step_time":
|
| 550 |
},
|
| 551 |
{
|
| 552 |
-
"clip_ratio/high_max": 0.
|
| 553 |
-
"clip_ratio/high_mean": 0.
|
| 554 |
-
"clip_ratio/low_mean": 0.
|
| 555 |
-
"clip_ratio/low_min": 0.
|
| 556 |
-
"clip_ratio/region_mean": 0.
|
| 557 |
-
"entropy":
|
| 558 |
"epoch": 1.25,
|
| 559 |
-
"grad_norm": 2.
|
| 560 |
-
"kl": 0.
|
| 561 |
"learning_rate": 2.1000000000000002e-06,
|
| 562 |
-
"loss": 0.
|
| 563 |
"step": 30,
|
| 564 |
-
"step_time": 0.
|
| 565 |
},
|
| 566 |
{
|
| 567 |
-
"clip_ratio/high_max": 0.
|
| 568 |
-
"clip_ratio/high_mean": 0.
|
| 569 |
-
"clip_ratio/low_mean": 0.
|
| 570 |
-
"clip_ratio/low_min": 0.
|
| 571 |
-
"clip_ratio/region_mean": 0.
|
| 572 |
-
"entropy": 1.
|
| 573 |
"epoch": 1.2916666666666667,
|
| 574 |
-
"grad_norm": 2.
|
| 575 |
-
"kl": 0.
|
| 576 |
"learning_rate": 2.0000000000000003e-06,
|
| 577 |
-
"loss":
|
| 578 |
"step": 31,
|
| 579 |
-
"step_time": 0.
|
| 580 |
},
|
| 581 |
{
|
| 582 |
-
"clip_ratio/high_max": 0.
|
| 583 |
-
"clip_ratio/high_mean": 0.
|
| 584 |
-
"clip_ratio/low_mean": 0.
|
| 585 |
-
"clip_ratio/low_min": 0.
|
| 586 |
-
"clip_ratio/region_mean": 0.
|
| 587 |
-
"entropy": 1.
|
| 588 |
"epoch": 1.3333333333333333,
|
| 589 |
-
"grad_norm":
|
| 590 |
-
"kl": 0.
|
| 591 |
"learning_rate": 1.9000000000000002e-06,
|
| 592 |
-
"loss": -0.
|
| 593 |
"step": 32,
|
| 594 |
-
"step_time": 0.
|
| 595 |
},
|
| 596 |
{
|
| 597 |
"clip_ratio/high_max": 0.0,
|
|
@@ -606,65 +606,65 @@
|
|
| 606 |
"completions/mean_terminated_length": 0.0,
|
| 607 |
"completions/min_length": 384.0,
|
| 608 |
"completions/min_terminated_length": 0.0,
|
| 609 |
-
"entropy":
|
| 610 |
"epoch": 1.375,
|
| 611 |
"frac_reward_zero_std": 0.0,
|
| 612 |
-
"grad_norm": 2.
|
| 613 |
-
"kl": 0.
|
| 614 |
"learning_rate": 1.8000000000000001e-06,
|
| 615 |
-
"loss": 0.
|
| 616 |
"num_tokens": 227556.0,
|
| 617 |
-
"reward": -0.
|
| 618 |
-
"reward_std": 0.
|
| 619 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 620 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 621 |
"step": 33,
|
| 622 |
-
"step_time": 12.
|
| 623 |
},
|
| 624 |
{
|
| 625 |
-
"clip_ratio/high_max": 0.
|
| 626 |
-
"clip_ratio/high_mean": 0.
|
| 627 |
-
"clip_ratio/low_mean": 0.
|
| 628 |
-
"clip_ratio/low_min": 0.
|
| 629 |
-
"clip_ratio/region_mean": 0.
|
| 630 |
-
"entropy": 1.
|
| 631 |
"epoch": 1.4166666666666667,
|
| 632 |
-
"grad_norm":
|
| 633 |
-
"kl": 0.
|
| 634 |
"learning_rate": 1.7000000000000002e-06,
|
| 635 |
-
"loss": -0.
|
| 636 |
"step": 34,
|
| 637 |
-
"step_time": 0.
|
| 638 |
},
|
| 639 |
{
|
| 640 |
-
"clip_ratio/high_max": 0.
|
| 641 |
-
"clip_ratio/high_mean": 0.
|
| 642 |
-
"clip_ratio/low_mean": 0.
|
| 643 |
-
"clip_ratio/low_min": 0.
|
| 644 |
-
"clip_ratio/region_mean": 0.
|
| 645 |
-
"entropy":
|
| 646 |
"epoch": 1.4583333333333333,
|
| 647 |
-
"grad_norm": 2.
|
| 648 |
-
"kl": 0.
|
| 649 |
"learning_rate": 1.6000000000000001e-06,
|
| 650 |
-
"loss": -0.
|
| 651 |
"step": 35,
|
| 652 |
-
"step_time": 0.
|
| 653 |
},
|
| 654 |
{
|
| 655 |
-
"clip_ratio/high_max": 0.
|
| 656 |
-
"clip_ratio/high_mean": 0.
|
| 657 |
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 658 |
"clip_ratio/low_min": 0.0032552082557231188,
|
| 659 |
-
"clip_ratio/region_mean": 0.
|
| 660 |
-
"entropy":
|
| 661 |
"epoch": 1.5,
|
| 662 |
-
"grad_norm": 1.
|
| 663 |
-
"kl": 0.
|
| 664 |
"learning_rate": 1.5e-06,
|
| 665 |
-
"loss": 0.
|
| 666 |
"step": 36,
|
| 667 |
-
"step_time": 0.
|
| 668 |
},
|
| 669 |
{
|
| 670 |
"clip_ratio/high_max": 0.0,
|
|
@@ -679,65 +679,65 @@
|
|
| 679 |
"completions/mean_terminated_length": 0.0,
|
| 680 |
"completions/min_length": 384.0,
|
| 681 |
"completions/min_terminated_length": 0.0,
|
| 682 |
-
"entropy": 1.
|
| 683 |
"epoch": 1.5416666666666665,
|
| 684 |
-
"frac_reward_zero_std": 0.
|
| 685 |
-
"grad_norm":
|
| 686 |
-
"kl": 0.
|
| 687 |
"learning_rate": 1.4000000000000001e-06,
|
| 688 |
-
"loss": -0.
|
| 689 |
"num_tokens": 252640.0,
|
| 690 |
-
"reward": -0.
|
| 691 |
-
"reward_std": 0.
|
| 692 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 693 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 694 |
"step": 37,
|
| 695 |
-
"step_time":
|
| 696 |
},
|
| 697 |
{
|
| 698 |
-
"clip_ratio/high_max": 0.
|
| 699 |
-
"clip_ratio/high_mean": 0.
|
| 700 |
-
"clip_ratio/low_mean": 0.
|
| 701 |
-
"clip_ratio/low_min": 0.
|
| 702 |
-
"clip_ratio/region_mean": 0.
|
| 703 |
-
"entropy": 1.
|
| 704 |
"epoch": 1.5833333333333335,
|
| 705 |
-
"grad_norm": 1.
|
| 706 |
-
"kl": 0.
|
| 707 |
"learning_rate": 1.3e-06,
|
| 708 |
-
"loss": 0.
|
| 709 |
"step": 38,
|
| 710 |
-
"step_time": 0.
|
| 711 |
},
|
| 712 |
{
|
| 713 |
-
"clip_ratio/high_max": 0.
|
| 714 |
-
"clip_ratio/high_mean": 0.
|
| 715 |
-
"clip_ratio/low_mean": 0.
|
| 716 |
-
"clip_ratio/low_min": 0.
|
| 717 |
-
"clip_ratio/region_mean": 0.
|
| 718 |
-
"entropy": 1.
|
| 719 |
"epoch": 1.625,
|
| 720 |
-
"grad_norm":
|
| 721 |
-
"kl": 0.
|
| 722 |
"learning_rate": 1.2000000000000002e-06,
|
| 723 |
-
"loss": 0.
|
| 724 |
"step": 39,
|
| 725 |
-
"step_time": 0.
|
| 726 |
},
|
| 727 |
{
|
| 728 |
-
"clip_ratio/high_max": 0.
|
| 729 |
-
"clip_ratio/high_mean": 0.
|
| 730 |
-
"clip_ratio/low_mean": 0.
|
| 731 |
-
"clip_ratio/low_min": 0.
|
| 732 |
-
"clip_ratio/region_mean": 0.
|
| 733 |
-
"entropy":
|
| 734 |
"epoch": 1.6666666666666665,
|
| 735 |
-
"grad_norm": 2.
|
| 736 |
-
"kl": 0.
|
| 737 |
"learning_rate": 1.1e-06,
|
| 738 |
-
"loss": -0.
|
| 739 |
"step": 40,
|
| 740 |
-
"step_time": 0.
|
| 741 |
}
|
| 742 |
],
|
| 743 |
"logging_steps": 1,
|
|
|
|
| 25 |
"entropy": 1.9362258911132812,
|
| 26 |
"epoch": 0.041666666666666664,
|
| 27 |
"frac_reward_zero_std": 0.0,
|
| 28 |
+
"grad_norm": 3.3426833152770996,
|
| 29 |
"kl": 0.0005498419050127268,
|
| 30 |
"learning_rate": 5e-06,
|
| 31 |
"loss": 0.13995857536792755,
|
|
|
|
| 35 |
"rewards/GeneratorRewardFunction/mean": -0.4352343678474426,
|
| 36 |
"rewards/GeneratorRewardFunction/std": 0.306624174118042,
|
| 37 |
"step": 1,
|
| 38 |
+
"step_time": 12.578062469000088
|
| 39 |
},
|
| 40 |
{
|
| 41 |
+
"clip_ratio/high_max": 0.00390625,
|
| 42 |
+
"clip_ratio/high_mean": 0.00390625,
|
| 43 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 44 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 45 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 46 |
+
"entropy": 1.2686206102371216,
|
| 47 |
"epoch": 0.08333333333333333,
|
| 48 |
+
"grad_norm": 2.8547239303588867,
|
| 49 |
+
"kl": 0.001546451705507934,
|
| 50 |
"learning_rate": 4.9000000000000005e-06,
|
| 51 |
+
"loss": -0.06681232899427414,
|
| 52 |
"step": 2,
|
| 53 |
+
"step_time": 0.22036709600001814
|
| 54 |
},
|
| 55 |
{
|
| 56 |
+
"clip_ratio/high_max": 0.012369791977107525,
|
| 57 |
+
"clip_ratio/high_mean": 0.012369791977107525,
|
| 58 |
+
"clip_ratio/low_mean": 0.015625,
|
| 59 |
+
"clip_ratio/low_min": 0.015625,
|
| 60 |
"clip_ratio/region_mean": 0.02799479104578495,
|
| 61 |
+
"entropy": 1.8668650388717651,
|
| 62 |
"epoch": 0.125,
|
| 63 |
+
"grad_norm": 2.4686105251312256,
|
| 64 |
+
"kl": 0.005345983896404505,
|
| 65 |
"learning_rate": 4.800000000000001e-06,
|
| 66 |
+
"loss": 0.010777520947158337,
|
| 67 |
"step": 3,
|
| 68 |
+
"step_time": 0.2199646679999887
|
| 69 |
},
|
| 70 |
{
|
| 71 |
+
"clip_ratio/high_max": 0.02083333395421505,
|
| 72 |
+
"clip_ratio/high_mean": 0.02083333395421505,
|
| 73 |
"clip_ratio/low_mean": 0.010416666977107525,
|
| 74 |
"clip_ratio/low_min": 0.010416666977107525,
|
| 75 |
+
"clip_ratio/region_mean": 0.03125,
|
| 76 |
+
"entropy": 1.1842881441116333,
|
| 77 |
"epoch": 0.16666666666666666,
|
| 78 |
+
"grad_norm": 1.569398045539856,
|
| 79 |
+
"kl": 0.0072342646308243275,
|
| 80 |
"learning_rate": 4.7e-06,
|
| 81 |
+
"loss": -0.08198019117116928,
|
| 82 |
"step": 4,
|
| 83 |
+
"step_time": 0.2201611520000597
|
| 84 |
},
|
| 85 |
{
|
| 86 |
+
"clip_ratio/high_max": 0.001953125,
|
| 87 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 88 |
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 89 |
"clip_ratio/low_min": 0.0006510416860692203,
|
| 90 |
+
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 91 |
"completions/clipped_ratio": 1.0,
|
| 92 |
"completions/max_length": 384.0,
|
| 93 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 95 |
"completions/mean_terminated_length": 0.0,
|
| 96 |
"completions/min_length": 384.0,
|
| 97 |
"completions/min_terminated_length": 0.0,
|
| 98 |
+
"entropy": 1.3128995895385742,
|
| 99 |
"epoch": 0.20833333333333334,
|
| 100 |
"frac_reward_zero_std": 0.0,
|
| 101 |
+
"grad_norm": 2.202421188354492,
|
| 102 |
+
"kl": 0.0129135362803936,
|
| 103 |
"learning_rate": 4.600000000000001e-06,
|
| 104 |
+
"loss": -0.0841849148273468,
|
| 105 |
"num_tokens": 50440.0,
|
| 106 |
+
"reward": -0.3341406285762787,
|
| 107 |
+
"reward_std": 0.3155691623687744,
|
| 108 |
+
"rewards/GeneratorRewardFunction/mean": -0.3341406285762787,
|
| 109 |
+
"rewards/GeneratorRewardFunction/std": 0.3155691623687744,
|
| 110 |
"step": 5,
|
| 111 |
+
"step_time": 12.076087126999937
|
| 112 |
},
|
| 113 |
{
|
| 114 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 115 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 116 |
+
"clip_ratio/low_mean": 0.0052083334885537624,
|
| 117 |
+
"clip_ratio/low_min": 0.0052083334885537624,
|
| 118 |
+
"clip_ratio/region_mean": 0.0078125,
|
| 119 |
+
"entropy": 1.3001914024353027,
|
| 120 |
"epoch": 0.25,
|
| 121 |
+
"grad_norm": 2.854139804840088,
|
| 122 |
+
"kl": 0.01436698716133833,
|
| 123 |
"learning_rate": 4.5e-06,
|
| 124 |
+
"loss": 0.02869725041091442,
|
| 125 |
"step": 6,
|
| 126 |
+
"step_time": 0.22715311399997518
|
| 127 |
},
|
| 128 |
{
|
| 129 |
+
"clip_ratio/high_max": 0.0071614584885537624,
|
| 130 |
+
"clip_ratio/high_mean": 0.0071614584885537624,
|
| 131 |
+
"clip_ratio/low_mean": 0.0071614584885537624,
|
| 132 |
+
"clip_ratio/low_min": 0.0071614584885537624,
|
| 133 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 134 |
+
"entropy": 1.0331100225448608,
|
| 135 |
"epoch": 0.2916666666666667,
|
| 136 |
+
"grad_norm": 1.9297211170196533,
|
| 137 |
+
"kl": 0.01791433058679104,
|
| 138 |
"learning_rate": 4.4e-06,
|
| 139 |
+
"loss": -0.028683962300419807,
|
| 140 |
"step": 7,
|
| 141 |
+
"step_time": 0.22586069900000894
|
| 142 |
},
|
| 143 |
{
|
| 144 |
"clip_ratio/high_max": 0.029296875,
|
| 145 |
"clip_ratio/high_mean": 0.029296875,
|
| 146 |
+
"clip_ratio/low_mean": 0.01171875,
|
| 147 |
+
"clip_ratio/low_min": 0.01171875,
|
| 148 |
+
"clip_ratio/region_mean": 0.041015625,
|
| 149 |
+
"entropy": 1.1462408304214478,
|
| 150 |
"epoch": 0.3333333333333333,
|
| 151 |
+
"grad_norm": 2.57124924659729,
|
| 152 |
+
"kl": 0.0388585664331913,
|
| 153 |
"learning_rate": 4.3e-06,
|
| 154 |
+
"loss": 0.08592668920755386,
|
| 155 |
"step": 8,
|
| 156 |
+
"step_time": 0.22552726200001416
|
| 157 |
},
|
| 158 |
{
|
| 159 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 168 |
"completions/mean_terminated_length": 0.0,
|
| 169 |
"completions/min_length": 384.0,
|
| 170 |
"completions/min_terminated_length": 0.0,
|
| 171 |
+
"entropy": 0.999983549118042,
|
| 172 |
"epoch": 0.375,
|
| 173 |
"frac_reward_zero_std": 0.25,
|
| 174 |
+
"grad_norm": 2.0052192211151123,
|
| 175 |
+
"kl": 0.030047910287976265,
|
| 176 |
"learning_rate": 4.2000000000000004e-06,
|
| 177 |
+
"loss": -0.05663062259554863,
|
| 178 |
"num_tokens": 75884.0,
|
| 179 |
+
"reward": -0.3902343511581421,
|
| 180 |
+
"reward_std": 0.31722894310951233,
|
| 181 |
+
"rewards/GeneratorRewardFunction/mean": -0.3902343511581421,
|
| 182 |
+
"rewards/GeneratorRewardFunction/std": 0.3172289729118347,
|
| 183 |
"step": 9,
|
| 184 |
+
"step_time": 12.047747722000054
|
| 185 |
},
|
| 186 |
{
|
| 187 |
+
"clip_ratio/high_max": 0.005859375,
|
| 188 |
+
"clip_ratio/high_mean": 0.005859375,
|
| 189 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 190 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 191 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 192 |
+
"entropy": 1.6177984476089478,
|
| 193 |
"epoch": 0.4166666666666667,
|
| 194 |
+
"grad_norm": 2.137237071990967,
|
| 195 |
+
"kl": 0.04101690649986267,
|
| 196 |
"learning_rate": 4.1e-06,
|
| 197 |
+
"loss": -0.02161034755408764,
|
| 198 |
"step": 10,
|
| 199 |
+
"step_time": 0.2252395229999138
|
| 200 |
},
|
| 201 |
{
|
| 202 |
+
"clip_ratio/high_max": 0.01692708395421505,
|
| 203 |
+
"clip_ratio/high_mean": 0.01692708395421505,
|
| 204 |
+
"clip_ratio/low_mean": 0.0065104165114462376,
|
| 205 |
+
"clip_ratio/low_min": 0.0065104165114462376,
|
| 206 |
+
"clip_ratio/region_mean": 0.0234375,
|
| 207 |
+
"entropy": 1.038699746131897,
|
| 208 |
"epoch": 0.4583333333333333,
|
| 209 |
+
"grad_norm": 2.672621965408325,
|
| 210 |
+
"kl": 0.031740155071020126,
|
| 211 |
"learning_rate": 4.000000000000001e-06,
|
| 212 |
+
"loss": 0.056199509650468826,
|
| 213 |
"step": 11,
|
| 214 |
+
"step_time": 0.22556489999999485
|
| 215 |
},
|
| 216 |
{
|
| 217 |
+
"clip_ratio/high_max": 0.01822916604578495,
|
| 218 |
+
"clip_ratio/high_mean": 0.01822916604578495,
|
| 219 |
+
"clip_ratio/low_mean": 0.010416666977107525,
|
| 220 |
+
"clip_ratio/low_min": 0.010416666977107525,
|
| 221 |
+
"clip_ratio/region_mean": 0.02864583395421505,
|
| 222 |
+
"entropy": 1.296442985534668,
|
| 223 |
"epoch": 0.5,
|
| 224 |
+
"grad_norm": 1.4488099813461304,
|
| 225 |
+
"kl": 0.04755128547549248,
|
| 226 |
"learning_rate": 3.900000000000001e-06,
|
| 227 |
+
"loss": 0.02385079860687256,
|
| 228 |
"step": 12,
|
| 229 |
+
"step_time": 0.22498397900005784
|
| 230 |
},
|
| 231 |
{
|
| 232 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 233 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 234 |
+
"clip_ratio/low_mean": 0.0,
|
| 235 |
+
"clip_ratio/low_min": 0.0,
|
| 236 |
+
"clip_ratio/region_mean": 0.0013020833721384406,
|
| 237 |
"completions/clipped_ratio": 1.0,
|
| 238 |
"completions/max_length": 384.0,
|
| 239 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 241 |
"completions/mean_terminated_length": 0.0,
|
| 242 |
"completions/min_length": 384.0,
|
| 243 |
"completions/min_terminated_length": 0.0,
|
| 244 |
+
"entropy": 1.3575109243392944,
|
| 245 |
"epoch": 0.5416666666666666,
|
| 246 |
+
"frac_reward_zero_std": 0.0,
|
| 247 |
+
"grad_norm": 2.2914443016052246,
|
| 248 |
+
"kl": 0.06257984787225723,
|
| 249 |
"learning_rate": 3.8000000000000005e-06,
|
| 250 |
+
"loss": -0.10538653284311295,
|
| 251 |
"num_tokens": 101112.0,
|
| 252 |
+
"reward": -0.22843749821186066,
|
| 253 |
+
"reward_std": 0.294514924287796,
|
| 254 |
+
"rewards/GeneratorRewardFunction/mean": -0.22843749821186066,
|
| 255 |
+
"rewards/GeneratorRewardFunction/std": 0.2945149540901184,
|
| 256 |
"step": 13,
|
| 257 |
+
"step_time": 12.01501083200003
|
| 258 |
},
|
| 259 |
{
|
| 260 |
+
"clip_ratio/high_max": 0.001953125,
|
| 261 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 262 |
+
"clip_ratio/low_mean": 0.0,
|
| 263 |
+
"clip_ratio/low_min": 0.0,
|
| 264 |
+
"clip_ratio/region_mean": 0.001953125,
|
| 265 |
+
"entropy": 1.2918612957000732,
|
| 266 |
"epoch": 0.5833333333333334,
|
| 267 |
+
"grad_norm": 3.0368542671203613,
|
| 268 |
+
"kl": 0.04979195073246956,
|
| 269 |
"learning_rate": 3.7e-06,
|
| 270 |
+
"loss": -0.003113487036898732,
|
| 271 |
"step": 14,
|
| 272 |
+
"step_time": 0.21806825399994523
|
| 273 |
},
|
| 274 |
{
|
| 275 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 276 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 277 |
+
"clip_ratio/low_mean": 0.005859375,
|
| 278 |
+
"clip_ratio/low_min": 0.005859375,
|
| 279 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 280 |
+
"entropy": 1.1081053018569946,
|
| 281 |
"epoch": 0.625,
|
| 282 |
+
"grad_norm": 3.5923683643341064,
|
| 283 |
+
"kl": 0.06817911565303802,
|
| 284 |
"learning_rate": 3.6000000000000003e-06,
|
| 285 |
+
"loss": 0.15118412673473358,
|
| 286 |
"step": 15,
|
| 287 |
+
"step_time": 0.217887520999966
|
| 288 |
},
|
| 289 |
{
|
| 290 |
+
"clip_ratio/high_max": 0.02018229104578495,
|
| 291 |
+
"clip_ratio/high_mean": 0.02018229104578495,
|
| 292 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 293 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 294 |
+
"clip_ratio/region_mean": 0.02278645895421505,
|
| 295 |
+
"entropy": 1.0803831815719604,
|
| 296 |
"epoch": 0.6666666666666666,
|
| 297 |
+
"grad_norm": 1.789110541343689,
|
| 298 |
+
"kl": 0.056480005383491516,
|
| 299 |
"learning_rate": 3.5e-06,
|
| 300 |
+
"loss": -0.03890883922576904,
|
| 301 |
"step": 16,
|
| 302 |
+
"step_time": 0.21781940799996846
|
| 303 |
},
|
| 304 |
{
|
| 305 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 314 |
"completions/mean_terminated_length": 0.0,
|
| 315 |
"completions/min_length": 384.0,
|
| 316 |
"completions/min_terminated_length": 0.0,
|
| 317 |
+
"entropy": 0.8709045052528381,
|
| 318 |
"epoch": 0.7083333333333334,
|
| 319 |
"frac_reward_zero_std": 0.0,
|
| 320 |
+
"grad_norm": 1.329393982887268,
|
| 321 |
+
"kl": 0.06073950603604317,
|
| 322 |
"learning_rate": 3.4000000000000005e-06,
|
| 323 |
+
"loss": -0.11920400708913803,
|
| 324 |
"num_tokens": 126436.0,
|
| 325 |
+
"reward": -0.2240625023841858,
|
| 326 |
+
"reward_std": 0.2881968021392822,
|
| 327 |
+
"rewards/GeneratorRewardFunction/mean": -0.2240625023841858,
|
| 328 |
+
"rewards/GeneratorRewardFunction/std": 0.2881968021392822,
|
| 329 |
"step": 17,
|
| 330 |
+
"step_time": 12.08798373600007
|
| 331 |
},
|
| 332 |
{
|
| 333 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 334 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 335 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 336 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 337 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 338 |
+
"entropy": 1.083386778831482,
|
| 339 |
"epoch": 0.75,
|
| 340 |
+
"grad_norm": 1.343295931816101,
|
| 341 |
+
"kl": 0.0919194221496582,
|
| 342 |
"learning_rate": 3.3000000000000006e-06,
|
| 343 |
+
"loss": -0.007308408617973328,
|
| 344 |
"step": 18,
|
| 345 |
+
"step_time": 0.2253850289998809
|
| 346 |
},
|
| 347 |
{
|
| 348 |
+
"clip_ratio/high_max": 0.005859375,
|
| 349 |
+
"clip_ratio/high_mean": 0.005859375,
|
| 350 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 351 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 352 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 353 |
+
"entropy": 1.406662940979004,
|
| 354 |
"epoch": 0.7916666666666666,
|
| 355 |
+
"grad_norm": 3.3420534133911133,
|
| 356 |
+
"kl": 0.06450249999761581,
|
| 357 |
"learning_rate": 3.2000000000000003e-06,
|
| 358 |
+
"loss": 0.12472107261419296,
|
| 359 |
"step": 19,
|
| 360 |
+
"step_time": 0.22467486499999723
|
| 361 |
},
|
| 362 |
{
|
| 363 |
+
"clip_ratio/high_max": 0.011067708022892475,
|
| 364 |
+
"clip_ratio/high_mean": 0.011067708022892475,
|
| 365 |
+
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 366 |
+
"clip_ratio/low_min": 0.0032552082557231188,
|
| 367 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 368 |
+
"entropy": 1.6491953134536743,
|
| 369 |
"epoch": 0.8333333333333334,
|
| 370 |
+
"grad_norm": 3.3672103881835938,
|
| 371 |
+
"kl": 0.0773777961730957,
|
| 372 |
"learning_rate": 3.1000000000000004e-06,
|
| 373 |
+
"loss": 0.0035695277620106936,
|
| 374 |
"step": 20,
|
| 375 |
+
"step_time": 0.22389788999998927
|
| 376 |
},
|
| 377 |
{
|
| 378 |
+
"clip_ratio/high_max": 0.0006510416860692203,
|
| 379 |
+
"clip_ratio/high_mean": 0.0006510416860692203,
|
| 380 |
"clip_ratio/low_mean": 0.0,
|
| 381 |
"clip_ratio/low_min": 0.0,
|
| 382 |
+
"clip_ratio/region_mean": 0.0006510416860692203,
|
| 383 |
"completions/clipped_ratio": 1.0,
|
| 384 |
"completions/max_length": 384.0,
|
| 385 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 387 |
"completions/mean_terminated_length": 0.0,
|
| 388 |
"completions/min_length": 384.0,
|
| 389 |
"completions/min_terminated_length": 0.0,
|
| 390 |
+
"entropy": 1.4178005456924438,
|
| 391 |
"epoch": 0.875,
|
| 392 |
+
"frac_reward_zero_std": 0.0,
|
| 393 |
+
"grad_norm": 2.6333799362182617,
|
| 394 |
+
"kl": 0.07392226904630661,
|
| 395 |
"learning_rate": 3e-06,
|
| 396 |
+
"loss": -0.09101495891809464,
|
| 397 |
"num_tokens": 151596.0,
|
| 398 |
+
"reward": -0.15257811546325684,
|
| 399 |
+
"reward_std": 0.24854345619678497,
|
| 400 |
+
"rewards/GeneratorRewardFunction/mean": -0.15257811546325684,
|
| 401 |
+
"rewards/GeneratorRewardFunction/std": 0.24854345619678497,
|
| 402 |
"step": 21,
|
| 403 |
+
"step_time": 12.020297105999816
|
| 404 |
},
|
| 405 |
{
|
| 406 |
+
"clip_ratio/high_max": 0.0006510416860692203,
|
| 407 |
+
"clip_ratio/high_mean": 0.0006510416860692203,
|
| 408 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 409 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 410 |
+
"clip_ratio/region_mean": 0.001953125,
|
| 411 |
+
"entropy": 1.2036248445510864,
|
| 412 |
"epoch": 0.9166666666666666,
|
| 413 |
+
"grad_norm": 2.1499149799346924,
|
| 414 |
+
"kl": 0.0772874653339386,
|
| 415 |
"learning_rate": 2.9e-06,
|
| 416 |
+
"loss": 0.08120749890804291,
|
| 417 |
"step": 22,
|
| 418 |
+
"step_time": 0.21995178900010615
|
| 419 |
},
|
| 420 |
{
|
| 421 |
+
"clip_ratio/high_max": 0.0032552082557231188,
|
| 422 |
+
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 423 |
+
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 424 |
+
"clip_ratio/low_min": 0.0032552082557231188,
|
| 425 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 426 |
+
"entropy": 1.1966055631637573,
|
| 427 |
"epoch": 0.9583333333333334,
|
| 428 |
+
"grad_norm": 2.0064616203308105,
|
| 429 |
+
"kl": 0.07331382483243942,
|
| 430 |
"learning_rate": 2.8000000000000003e-06,
|
| 431 |
+
"loss": 0.03140506148338318,
|
| 432 |
"step": 23,
|
| 433 |
+
"step_time": 0.21996421700009705
|
| 434 |
},
|
| 435 |
{
|
| 436 |
+
"clip_ratio/high_max": 0.011067708022892475,
|
| 437 |
+
"clip_ratio/high_mean": 0.011067708022892475,
|
| 438 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 439 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 440 |
+
"clip_ratio/region_mean": 0.01171875,
|
| 441 |
+
"entropy": 0.9102082252502441,
|
| 442 |
"epoch": 1.0,
|
| 443 |
+
"grad_norm": 1.7175334692001343,
|
| 444 |
+
"kl": 0.14611481130123138,
|
| 445 |
"learning_rate": 2.7000000000000004e-06,
|
| 446 |
+
"loss": -0.021010393276810646,
|
| 447 |
"step": 24,
|
| 448 |
+
"step_time": 0.21931950600014716
|
| 449 |
},
|
| 450 |
{
|
| 451 |
+
"clip_ratio/high_max": 0.0,
|
| 452 |
+
"clip_ratio/high_mean": 0.0,
|
| 453 |
"clip_ratio/low_mean": 0.0,
|
| 454 |
"clip_ratio/low_min": 0.0,
|
| 455 |
+
"clip_ratio/region_mean": 0.0,
|
| 456 |
"completions/clipped_ratio": 1.0,
|
| 457 |
"completions/max_length": 384.0,
|
| 458 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 460 |
"completions/mean_terminated_length": 0.0,
|
| 461 |
"completions/min_length": 384.0,
|
| 462 |
"completions/min_terminated_length": 0.0,
|
| 463 |
+
"entropy": 1.9153881072998047,
|
| 464 |
"epoch": 1.0416666666666667,
|
| 465 |
"frac_reward_zero_std": 0.0,
|
| 466 |
+
"grad_norm": 2.3460445404052734,
|
| 467 |
+
"kl": 0.09710023552179337,
|
| 468 |
"learning_rate": 2.6e-06,
|
| 469 |
+
"loss": 0.015220091678202152,
|
| 470 |
"num_tokens": 177212.0,
|
| 471 |
+
"reward": -0.15656250715255737,
|
| 472 |
+
"reward_std": 0.22349287569522858,
|
| 473 |
+
"rewards/GeneratorRewardFunction/mean": -0.15656250715255737,
|
| 474 |
+
"rewards/GeneratorRewardFunction/std": 0.22349286079406738,
|
| 475 |
"step": 25,
|
| 476 |
+
"step_time": 12.153388549000056
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"clip_ratio/high_max": 0.00390625,
|
|
|
|
| 481 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 482 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 483 |
"clip_ratio/region_mean": 0.0052083334885537624,
|
| 484 |
+
"entropy": 1.365325927734375,
|
| 485 |
"epoch": 1.0833333333333333,
|
| 486 |
+
"grad_norm": 1.8710312843322754,
|
| 487 |
+
"kl": 0.0985046848654747,
|
| 488 |
"learning_rate": 2.5e-06,
|
| 489 |
+
"loss": -0.02838735282421112,
|
| 490 |
"step": 26,
|
| 491 |
+
"step_time": 0.22659933299996737
|
| 492 |
},
|
| 493 |
{
|
| 494 |
+
"clip_ratio/high_max": 0.008463541977107525,
|
| 495 |
+
"clip_ratio/high_mean": 0.008463541977107525,
|
| 496 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 497 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 498 |
+
"clip_ratio/region_mean": 0.009765625,
|
| 499 |
+
"entropy": 1.2517439126968384,
|
| 500 |
"epoch": 1.125,
|
| 501 |
+
"grad_norm": 2.821958303451538,
|
| 502 |
+
"kl": 0.09274079650640488,
|
| 503 |
"learning_rate": 2.4000000000000003e-06,
|
| 504 |
+
"loss": -0.007298170123249292,
|
| 505 |
"step": 27,
|
| 506 |
+
"step_time": 0.22647249999999985
|
| 507 |
},
|
| 508 |
{
|
| 509 |
+
"clip_ratio/high_max": 0.0052083334885537624,
|
| 510 |
+
"clip_ratio/high_mean": 0.0052083334885537624,
|
| 511 |
+
"clip_ratio/low_mean": 0.009114583022892475,
|
| 512 |
+
"clip_ratio/low_min": 0.009114583022892475,
|
| 513 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 514 |
+
"entropy": 2.0579044818878174,
|
| 515 |
"epoch": 1.1666666666666667,
|
| 516 |
+
"grad_norm": 3.259742259979248,
|
| 517 |
+
"kl": 0.10746321082115173,
|
| 518 |
"learning_rate": 2.3000000000000004e-06,
|
| 519 |
+
"loss": 0.021702758967876434,
|
| 520 |
"step": 28,
|
| 521 |
+
"step_time": 0.22640677999993386
|
| 522 |
},
|
| 523 |
{
|
| 524 |
+
"clip_ratio/high_max": 0.0,
|
| 525 |
+
"clip_ratio/high_mean": 0.0,
|
| 526 |
"clip_ratio/low_mean": 0.0,
|
| 527 |
"clip_ratio/low_min": 0.0,
|
| 528 |
+
"clip_ratio/region_mean": 0.0,
|
| 529 |
"completions/clipped_ratio": 1.0,
|
| 530 |
"completions/max_length": 384.0,
|
| 531 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 533 |
"completions/mean_terminated_length": 0.0,
|
| 534 |
"completions/min_length": 384.0,
|
| 535 |
"completions/min_terminated_length": 0.0,
|
| 536 |
+
"entropy": 1.3725861310958862,
|
| 537 |
"epoch": 1.2083333333333333,
|
| 538 |
"frac_reward_zero_std": 0.0,
|
| 539 |
+
"grad_norm": 1.8806989192962646,
|
| 540 |
+
"kl": 0.11961983889341354,
|
| 541 |
"learning_rate": 2.2e-06,
|
| 542 |
+
"loss": 0.07187109440565109,
|
| 543 |
"num_tokens": 202200.0,
|
| 544 |
+
"reward": -0.0561029389500618,
|
| 545 |
+
"reward_std": 0.314301997423172,
|
| 546 |
+
"rewards/GeneratorRewardFunction/mean": -0.0561029389500618,
|
| 547 |
+
"rewards/GeneratorRewardFunction/std": 0.314301997423172,
|
| 548 |
"step": 29,
|
| 549 |
+
"step_time": 13.662849896999887
|
| 550 |
},
|
| 551 |
{
|
| 552 |
+
"clip_ratio/high_max": 0.001953125,
|
| 553 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 554 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 555 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 556 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 557 |
+
"entropy": 1.2213298082351685,
|
| 558 |
"epoch": 1.25,
|
| 559 |
+
"grad_norm": 2.1918396949768066,
|
| 560 |
+
"kl": 0.12398240715265274,
|
| 561 |
"learning_rate": 2.1000000000000002e-06,
|
| 562 |
+
"loss": -0.052896980196237564,
|
| 563 |
"step": 30,
|
| 564 |
+
"step_time": 0.2210835590001352
|
| 565 |
},
|
| 566 |
{
|
| 567 |
+
"clip_ratio/high_max": 0.001953125,
|
| 568 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 569 |
+
"clip_ratio/low_mean": 0.001953125,
|
| 570 |
+
"clip_ratio/low_min": 0.001953125,
|
| 571 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 572 |
+
"entropy": 1.2683231830596924,
|
| 573 |
"epoch": 1.2916666666666667,
|
| 574 |
+
"grad_norm": 2.524726390838623,
|
| 575 |
+
"kl": 0.14297537505626678,
|
| 576 |
"learning_rate": 2.0000000000000003e-06,
|
| 577 |
+
"loss": 0.12745414674282074,
|
| 578 |
"step": 31,
|
| 579 |
+
"step_time": 0.22096665699996265
|
| 580 |
},
|
| 581 |
{
|
| 582 |
+
"clip_ratio/high_max": 0.0032552082557231188,
|
| 583 |
+
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 584 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 585 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 586 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 587 |
+
"entropy": 1.0583091974258423,
|
| 588 |
"epoch": 1.3333333333333333,
|
| 589 |
+
"grad_norm": 2.408073902130127,
|
| 590 |
+
"kl": 0.0881701335310936,
|
| 591 |
"learning_rate": 1.9000000000000002e-06,
|
| 592 |
+
"loss": -0.14430458843708038,
|
| 593 |
"step": 32,
|
| 594 |
+
"step_time": 0.21999255600007928
|
| 595 |
},
|
| 596 |
{
|
| 597 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 606 |
"completions/mean_terminated_length": 0.0,
|
| 607 |
"completions/min_length": 384.0,
|
| 608 |
"completions/min_terminated_length": 0.0,
|
| 609 |
+
"entropy": 1.3751106262207031,
|
| 610 |
"epoch": 1.375,
|
| 611 |
"frac_reward_zero_std": 0.0,
|
| 612 |
+
"grad_norm": 2.4002864360809326,
|
| 613 |
+
"kl": 0.0900505781173706,
|
| 614 |
"learning_rate": 1.8000000000000001e-06,
|
| 615 |
+
"loss": 0.06270528584718704,
|
| 616 |
"num_tokens": 227556.0,
|
| 617 |
+
"reward": -0.11414062231779099,
|
| 618 |
+
"reward_std": 0.21683935821056366,
|
| 619 |
+
"rewards/GeneratorRewardFunction/mean": -0.11414062231779099,
|
| 620 |
+
"rewards/GeneratorRewardFunction/std": 0.21683938801288605,
|
| 621 |
"step": 33,
|
| 622 |
+
"step_time": 12.070902493999938
|
| 623 |
},
|
| 624 |
{
|
| 625 |
+
"clip_ratio/high_max": 0.0045572915114462376,
|
| 626 |
+
"clip_ratio/high_mean": 0.0045572915114462376,
|
| 627 |
+
"clip_ratio/low_mean": 0.0,
|
| 628 |
+
"clip_ratio/low_min": 0.0,
|
| 629 |
+
"clip_ratio/region_mean": 0.0045572915114462376,
|
| 630 |
+
"entropy": 1.2606719732284546,
|
| 631 |
"epoch": 1.4166666666666667,
|
| 632 |
+
"grad_norm": 1.671729326248169,
|
| 633 |
+
"kl": 0.1210540160536766,
|
| 634 |
"learning_rate": 1.7000000000000002e-06,
|
| 635 |
+
"loss": -0.04401962831616402,
|
| 636 |
"step": 34,
|
| 637 |
+
"step_time": 0.2276347459999215
|
| 638 |
},
|
| 639 |
{
|
| 640 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 641 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 642 |
+
"clip_ratio/low_mean": 0.001953125,
|
| 643 |
+
"clip_ratio/low_min": 0.001953125,
|
| 644 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 645 |
+
"entropy": 1.2780500650405884,
|
| 646 |
"epoch": 1.4583333333333333,
|
| 647 |
+
"grad_norm": 2.278010845184326,
|
| 648 |
+
"kl": 0.11484409123659134,
|
| 649 |
"learning_rate": 1.6000000000000001e-06,
|
| 650 |
+
"loss": -0.08475238084793091,
|
| 651 |
"step": 35,
|
| 652 |
+
"step_time": 0.22882699100000536
|
| 653 |
},
|
| 654 |
{
|
| 655 |
+
"clip_ratio/high_max": 0.0052083334885537624,
|
| 656 |
+
"clip_ratio/high_mean": 0.0052083334885537624,
|
| 657 |
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 658 |
"clip_ratio/low_min": 0.0032552082557231188,
|
| 659 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 660 |
+
"entropy": 1.0553101301193237,
|
| 661 |
"epoch": 1.5,
|
| 662 |
+
"grad_norm": 1.582037091255188,
|
| 663 |
+
"kl": 0.12029703706502914,
|
| 664 |
"learning_rate": 1.5e-06,
|
| 665 |
+
"loss": 0.06627888232469559,
|
| 666 |
"step": 36,
|
| 667 |
+
"step_time": 0.22751581399984389
|
| 668 |
},
|
| 669 |
{
|
| 670 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 679 |
"completions/mean_terminated_length": 0.0,
|
| 680 |
"completions/min_length": 384.0,
|
| 681 |
"completions/min_terminated_length": 0.0,
|
| 682 |
+
"entropy": 1.0647958517074585,
|
| 683 |
"epoch": 1.5416666666666665,
|
| 684 |
+
"frac_reward_zero_std": 0.0,
|
| 685 |
+
"grad_norm": 2.1763558387756348,
|
| 686 |
+
"kl": 0.08708903193473816,
|
| 687 |
"learning_rate": 1.4000000000000001e-06,
|
| 688 |
+
"loss": -0.00017260713502764702,
|
| 689 |
"num_tokens": 252640.0,
|
| 690 |
+
"reward": -0.10210937261581421,
|
| 691 |
+
"reward_std": 0.19573244452476501,
|
| 692 |
+
"rewards/GeneratorRewardFunction/mean": -0.10210937261581421,
|
| 693 |
+
"rewards/GeneratorRewardFunction/std": 0.1957324594259262,
|
| 694 |
"step": 37,
|
| 695 |
+
"step_time": 12.015305628000078
|
| 696 |
},
|
| 697 |
{
|
| 698 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 699 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 700 |
+
"clip_ratio/low_mean": 0.0,
|
| 701 |
+
"clip_ratio/low_min": 0.0,
|
| 702 |
+
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 703 |
+
"entropy": 1.0041130781173706,
|
| 704 |
"epoch": 1.5833333333333335,
|
| 705 |
+
"grad_norm": 1.6093602180480957,
|
| 706 |
+
"kl": 0.11537543684244156,
|
| 707 |
"learning_rate": 1.3e-06,
|
| 708 |
+
"loss": -0.12453166395425797,
|
| 709 |
"step": 38,
|
| 710 |
+
"step_time": 0.22048816200003785
|
| 711 |
},
|
| 712 |
{
|
| 713 |
+
"clip_ratio/high_max": 0.0045572915114462376,
|
| 714 |
+
"clip_ratio/high_mean": 0.0045572915114462376,
|
| 715 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 716 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 717 |
+
"clip_ratio/region_mean": 0.005859375,
|
| 718 |
+
"entropy": 1.500306487083435,
|
| 719 |
"epoch": 1.625,
|
| 720 |
+
"grad_norm": 3.409069299697876,
|
| 721 |
+
"kl": 0.10904627293348312,
|
| 722 |
"learning_rate": 1.2000000000000002e-06,
|
| 723 |
+
"loss": 0.12661518156528473,
|
| 724 |
"step": 39,
|
| 725 |
+
"step_time": 0.22087437000004684
|
| 726 |
},
|
| 727 |
{
|
| 728 |
+
"clip_ratio/high_max": 0.0078125,
|
| 729 |
+
"clip_ratio/high_mean": 0.0078125,
|
| 730 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 731 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 732 |
+
"clip_ratio/region_mean": 0.009114583022892475,
|
| 733 |
+
"entropy": 1.0560635328292847,
|
| 734 |
"epoch": 1.6666666666666665,
|
| 735 |
+
"grad_norm": 2.0718417167663574,
|
| 736 |
+
"kl": 0.11926760524511337,
|
| 737 |
"learning_rate": 1.1e-06,
|
| 738 |
+
"loss": -0.0004449083062354475,
|
| 739 |
"step": 40,
|
| 740 |
+
"step_time": 0.2202887500000088
|
| 741 |
}
|
| 742 |
],
|
| 743 |
"logging_steps": 1,
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1976163472
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6fa4eed67a84ce4076ba3848a078496971cd34ba048c794e52cc3b4aab54a27
|
| 3 |
size 1976163472
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/optimizer.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 3952509771
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b145cc09cf03081708247bd99e0dd46e23f798d922e5e7df9e75880345e1d969
|
| 3 |
size 3952509771
|
self_play_hf_a10g_train/round_001/generator_train/checkpoint-50/trainer_state.json
CHANGED
|
@@ -25,7 +25,7 @@
|
|
| 25 |
"entropy": 1.9362258911132812,
|
| 26 |
"epoch": 0.041666666666666664,
|
| 27 |
"frac_reward_zero_std": 0.0,
|
| 28 |
-
"grad_norm": 3.
|
| 29 |
"kl": 0.0005498419050127268,
|
| 30 |
"learning_rate": 5e-06,
|
| 31 |
"loss": 0.13995857536792755,
|
|
@@ -35,59 +35,59 @@
|
|
| 35 |
"rewards/GeneratorRewardFunction/mean": -0.4352343678474426,
|
| 36 |
"rewards/GeneratorRewardFunction/std": 0.306624174118042,
|
| 37 |
"step": 1,
|
| 38 |
-
"step_time": 12.
|
| 39 |
},
|
| 40 |
{
|
| 41 |
-
"clip_ratio/high_max": 0.
|
| 42 |
-
"clip_ratio/high_mean": 0.
|
| 43 |
-
"clip_ratio/low_mean": 0.
|
| 44 |
-
"clip_ratio/low_min": 0.
|
| 45 |
-
"clip_ratio/region_mean": 0.
|
| 46 |
-
"entropy": 1.
|
| 47 |
"epoch": 0.08333333333333333,
|
| 48 |
-
"grad_norm": 2.
|
| 49 |
-
"kl": 0.
|
| 50 |
"learning_rate": 4.9000000000000005e-06,
|
| 51 |
-
"loss": -0.
|
| 52 |
"step": 2,
|
| 53 |
-
"step_time": 0.
|
| 54 |
},
|
| 55 |
{
|
| 56 |
-
"clip_ratio/high_max": 0.
|
| 57 |
-
"clip_ratio/high_mean": 0.
|
| 58 |
-
"clip_ratio/low_mean": 0.
|
| 59 |
-
"clip_ratio/low_min": 0.
|
| 60 |
"clip_ratio/region_mean": 0.02799479104578495,
|
| 61 |
-
"entropy": 1.
|
| 62 |
"epoch": 0.125,
|
| 63 |
-
"grad_norm": 2.
|
| 64 |
-
"kl": 0.
|
| 65 |
"learning_rate": 4.800000000000001e-06,
|
| 66 |
-
"loss": 0.
|
| 67 |
"step": 3,
|
| 68 |
-
"step_time": 0.
|
| 69 |
},
|
| 70 |
{
|
| 71 |
-
"clip_ratio/high_max": 0.
|
| 72 |
-
"clip_ratio/high_mean": 0.
|
| 73 |
"clip_ratio/low_mean": 0.010416666977107525,
|
| 74 |
"clip_ratio/low_min": 0.010416666977107525,
|
| 75 |
-
"clip_ratio/region_mean": 0.
|
| 76 |
-
"entropy": 1.
|
| 77 |
"epoch": 0.16666666666666666,
|
| 78 |
-
"grad_norm": 1.
|
| 79 |
-
"kl": 0.
|
| 80 |
"learning_rate": 4.7e-06,
|
| 81 |
-
"loss": -0.
|
| 82 |
"step": 4,
|
| 83 |
-
"step_time": 0.
|
| 84 |
},
|
| 85 |
{
|
| 86 |
-
"clip_ratio/high_max": 0.
|
| 87 |
-
"clip_ratio/high_mean": 0.
|
| 88 |
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 89 |
"clip_ratio/low_min": 0.0006510416860692203,
|
| 90 |
-
"clip_ratio/region_mean": 0.
|
| 91 |
"completions/clipped_ratio": 1.0,
|
| 92 |
"completions/max_length": 384.0,
|
| 93 |
"completions/max_terminated_length": 0.0,
|
|
@@ -95,65 +95,65 @@
|
|
| 95 |
"completions/mean_terminated_length": 0.0,
|
| 96 |
"completions/min_length": 384.0,
|
| 97 |
"completions/min_terminated_length": 0.0,
|
| 98 |
-
"entropy": 1.
|
| 99 |
"epoch": 0.20833333333333334,
|
| 100 |
"frac_reward_zero_std": 0.0,
|
| 101 |
-
"grad_norm":
|
| 102 |
-
"kl": 0.
|
| 103 |
"learning_rate": 4.600000000000001e-06,
|
| 104 |
-
"loss": 0.
|
| 105 |
"num_tokens": 50440.0,
|
| 106 |
-
"reward": -0.
|
| 107 |
-
"reward_std": 0.
|
| 108 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 109 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 110 |
"step": 5,
|
| 111 |
-
"step_time": 12.
|
| 112 |
},
|
| 113 |
{
|
| 114 |
-
"clip_ratio/high_max": 0.
|
| 115 |
-
"clip_ratio/high_mean": 0.
|
| 116 |
-
"clip_ratio/low_mean": 0.
|
| 117 |
-
"clip_ratio/low_min": 0.
|
| 118 |
-
"clip_ratio/region_mean": 0.
|
| 119 |
-
"entropy": 1.
|
| 120 |
"epoch": 0.25,
|
| 121 |
-
"grad_norm": 2.
|
| 122 |
-
"kl": 0.
|
| 123 |
"learning_rate": 4.5e-06,
|
| 124 |
-
"loss":
|
| 125 |
"step": 6,
|
| 126 |
-
"step_time": 0.
|
| 127 |
},
|
| 128 |
{
|
| 129 |
-
"clip_ratio/high_max": 0.
|
| 130 |
-
"clip_ratio/high_mean": 0.
|
| 131 |
-
"clip_ratio/low_mean": 0.
|
| 132 |
-
"clip_ratio/low_min": 0.
|
| 133 |
-
"clip_ratio/region_mean": 0.
|
| 134 |
-
"entropy": 1.
|
| 135 |
"epoch": 0.2916666666666667,
|
| 136 |
-
"grad_norm":
|
| 137 |
-
"kl": 0.
|
| 138 |
"learning_rate": 4.4e-06,
|
| 139 |
-
"loss": 0.
|
| 140 |
"step": 7,
|
| 141 |
-
"step_time": 0.
|
| 142 |
},
|
| 143 |
{
|
| 144 |
"clip_ratio/high_max": 0.029296875,
|
| 145 |
"clip_ratio/high_mean": 0.029296875,
|
| 146 |
-
"clip_ratio/low_mean": 0.
|
| 147 |
-
"clip_ratio/low_min": 0.
|
| 148 |
-
"clip_ratio/region_mean": 0.
|
| 149 |
-
"entropy":
|
| 150 |
"epoch": 0.3333333333333333,
|
| 151 |
-
"grad_norm":
|
| 152 |
-
"kl": 0.
|
| 153 |
"learning_rate": 4.3e-06,
|
| 154 |
-
"loss":
|
| 155 |
"step": 8,
|
| 156 |
-
"step_time": 0.
|
| 157 |
},
|
| 158 |
{
|
| 159 |
"clip_ratio/high_max": 0.0,
|
|
@@ -168,72 +168,72 @@
|
|
| 168 |
"completions/mean_terminated_length": 0.0,
|
| 169 |
"completions/min_length": 384.0,
|
| 170 |
"completions/min_terminated_length": 0.0,
|
| 171 |
-
"entropy":
|
| 172 |
"epoch": 0.375,
|
| 173 |
"frac_reward_zero_std": 0.25,
|
| 174 |
-
"grad_norm":
|
| 175 |
-
"kl": 0.
|
| 176 |
"learning_rate": 4.2000000000000004e-06,
|
| 177 |
-
"loss": -0.
|
| 178 |
"num_tokens": 75884.0,
|
| 179 |
-
"reward": -0.
|
| 180 |
-
"reward_std": 0.
|
| 181 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 182 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 183 |
"step": 9,
|
| 184 |
-
"step_time":
|
| 185 |
},
|
| 186 |
{
|
| 187 |
-
"clip_ratio/high_max": 0.
|
| 188 |
-
"clip_ratio/high_mean": 0.
|
| 189 |
-
"clip_ratio/low_mean": 0.
|
| 190 |
-
"clip_ratio/low_min": 0.
|
| 191 |
-
"clip_ratio/region_mean": 0.
|
| 192 |
-
"entropy": 1.
|
| 193 |
"epoch": 0.4166666666666667,
|
| 194 |
-
"grad_norm": 2.
|
| 195 |
-
"kl": 0.
|
| 196 |
"learning_rate": 4.1e-06,
|
| 197 |
-
"loss": 0.
|
| 198 |
"step": 10,
|
| 199 |
-
"step_time": 0.
|
| 200 |
},
|
| 201 |
{
|
| 202 |
-
"clip_ratio/high_max": 0.
|
| 203 |
-
"clip_ratio/high_mean": 0.
|
| 204 |
-
"clip_ratio/low_mean": 0.
|
| 205 |
-
"clip_ratio/low_min": 0.
|
| 206 |
-
"clip_ratio/region_mean": 0.
|
| 207 |
-
"entropy":
|
| 208 |
"epoch": 0.4583333333333333,
|
| 209 |
-
"grad_norm": 2.
|
| 210 |
-
"kl": 0.
|
| 211 |
"learning_rate": 4.000000000000001e-06,
|
| 212 |
-
"loss": 0.
|
| 213 |
"step": 11,
|
| 214 |
-
"step_time": 0.
|
| 215 |
},
|
| 216 |
{
|
| 217 |
-
"clip_ratio/high_max": 0.
|
| 218 |
-
"clip_ratio/high_mean": 0.
|
| 219 |
-
"clip_ratio/low_mean": 0.
|
| 220 |
-
"clip_ratio/low_min": 0.
|
| 221 |
-
"clip_ratio/region_mean": 0.
|
| 222 |
-
"entropy": 1.
|
| 223 |
"epoch": 0.5,
|
| 224 |
-
"grad_norm":
|
| 225 |
-
"kl": 0.
|
| 226 |
"learning_rate": 3.900000000000001e-06,
|
| 227 |
-
"loss":
|
| 228 |
"step": 12,
|
| 229 |
-
"step_time": 0.
|
| 230 |
},
|
| 231 |
{
|
| 232 |
-
"clip_ratio/high_max": 0.
|
| 233 |
-
"clip_ratio/high_mean": 0.
|
| 234 |
-
"clip_ratio/low_mean": 0.
|
| 235 |
-
"clip_ratio/low_min": 0.
|
| 236 |
-
"clip_ratio/region_mean": 0.
|
| 237 |
"completions/clipped_ratio": 1.0,
|
| 238 |
"completions/max_length": 384.0,
|
| 239 |
"completions/max_terminated_length": 0.0,
|
|
@@ -241,65 +241,65 @@
|
|
| 241 |
"completions/mean_terminated_length": 0.0,
|
| 242 |
"completions/min_length": 384.0,
|
| 243 |
"completions/min_terminated_length": 0.0,
|
| 244 |
-
"entropy": 1.
|
| 245 |
"epoch": 0.5416666666666666,
|
| 246 |
-
"frac_reward_zero_std": 0.
|
| 247 |
-
"grad_norm": 2.
|
| 248 |
-
"kl": 0.
|
| 249 |
"learning_rate": 3.8000000000000005e-06,
|
| 250 |
-
"loss": 0.
|
| 251 |
"num_tokens": 101112.0,
|
| 252 |
-
"reward": -0.
|
| 253 |
-
"reward_std": 0.
|
| 254 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 255 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 256 |
"step": 13,
|
| 257 |
-
"step_time": 12.
|
| 258 |
},
|
| 259 |
{
|
| 260 |
-
"clip_ratio/high_max": 0.
|
| 261 |
-
"clip_ratio/high_mean": 0.
|
| 262 |
-
"clip_ratio/low_mean": 0.
|
| 263 |
-
"clip_ratio/low_min": 0.
|
| 264 |
-
"clip_ratio/region_mean": 0.
|
| 265 |
-
"entropy": 1.
|
| 266 |
"epoch": 0.5833333333333334,
|
| 267 |
-
"grad_norm":
|
| 268 |
-
"kl": 0.
|
| 269 |
"learning_rate": 3.7e-06,
|
| 270 |
-
"loss": 0.
|
| 271 |
"step": 14,
|
| 272 |
-
"step_time": 0.
|
| 273 |
},
|
| 274 |
{
|
| 275 |
-
"clip_ratio/high_max": 0.
|
| 276 |
-
"clip_ratio/high_mean": 0.
|
| 277 |
-
"clip_ratio/low_mean": 0.
|
| 278 |
-
"clip_ratio/low_min": 0.
|
| 279 |
-
"clip_ratio/region_mean": 0.
|
| 280 |
-
"entropy": 1.
|
| 281 |
"epoch": 0.625,
|
| 282 |
-
"grad_norm":
|
| 283 |
-
"kl": 0.
|
| 284 |
"learning_rate": 3.6000000000000003e-06,
|
| 285 |
-
"loss":
|
| 286 |
"step": 15,
|
| 287 |
-
"step_time": 0.
|
| 288 |
},
|
| 289 |
{
|
| 290 |
-
"clip_ratio/high_max": 0.
|
| 291 |
-
"clip_ratio/high_mean": 0.
|
| 292 |
-
"clip_ratio/low_mean": 0.
|
| 293 |
-
"clip_ratio/low_min": 0.
|
| 294 |
-
"clip_ratio/region_mean": 0.
|
| 295 |
-
"entropy":
|
| 296 |
"epoch": 0.6666666666666666,
|
| 297 |
-
"grad_norm":
|
| 298 |
-
"kl": 0.
|
| 299 |
"learning_rate": 3.5e-06,
|
| 300 |
-
"loss": -0.
|
| 301 |
"step": 16,
|
| 302 |
-
"step_time": 0.
|
| 303 |
},
|
| 304 |
{
|
| 305 |
"clip_ratio/high_max": 0.0,
|
|
@@ -314,72 +314,72 @@
|
|
| 314 |
"completions/mean_terminated_length": 0.0,
|
| 315 |
"completions/min_length": 384.0,
|
| 316 |
"completions/min_terminated_length": 0.0,
|
| 317 |
-
"entropy":
|
| 318 |
"epoch": 0.7083333333333334,
|
| 319 |
"frac_reward_zero_std": 0.0,
|
| 320 |
-
"grad_norm":
|
| 321 |
-
"kl": 0.
|
| 322 |
"learning_rate": 3.4000000000000005e-06,
|
| 323 |
-
"loss": 0.
|
| 324 |
"num_tokens": 126436.0,
|
| 325 |
-
"reward": -0.
|
| 326 |
-
"reward_std": 0.
|
| 327 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 328 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 329 |
"step": 17,
|
| 330 |
-
"step_time": 12.
|
| 331 |
},
|
| 332 |
{
|
| 333 |
-
"clip_ratio/high_max": 0.
|
| 334 |
-
"clip_ratio/high_mean": 0.
|
| 335 |
-
"clip_ratio/low_mean": 0.
|
| 336 |
-
"clip_ratio/low_min": 0.
|
| 337 |
-
"clip_ratio/region_mean": 0.
|
| 338 |
-
"entropy":
|
| 339 |
"epoch": 0.75,
|
| 340 |
-
"grad_norm":
|
| 341 |
-
"kl": 0.
|
| 342 |
"learning_rate": 3.3000000000000006e-06,
|
| 343 |
-
"loss": -0.
|
| 344 |
"step": 18,
|
| 345 |
-
"step_time": 0.
|
| 346 |
},
|
| 347 |
{
|
| 348 |
-
"clip_ratio/high_max": 0.
|
| 349 |
-
"clip_ratio/high_mean": 0.
|
| 350 |
-
"clip_ratio/low_mean": 0.
|
| 351 |
-
"clip_ratio/low_min": 0.
|
| 352 |
-
"clip_ratio/region_mean": 0.
|
| 353 |
-
"entropy":
|
| 354 |
"epoch": 0.7916666666666666,
|
| 355 |
-
"grad_norm":
|
| 356 |
-
"kl": 0.
|
| 357 |
"learning_rate": 3.2000000000000003e-06,
|
| 358 |
-
"loss":
|
| 359 |
"step": 19,
|
| 360 |
-
"step_time": 0.
|
| 361 |
},
|
| 362 |
{
|
| 363 |
-
"clip_ratio/high_max": 0.
|
| 364 |
-
"clip_ratio/high_mean": 0.
|
| 365 |
-
"clip_ratio/low_mean": 0.
|
| 366 |
-
"clip_ratio/low_min": 0.
|
| 367 |
-
"clip_ratio/region_mean": 0.
|
| 368 |
-
"entropy": 1.
|
| 369 |
"epoch": 0.8333333333333334,
|
| 370 |
-
"grad_norm":
|
| 371 |
-
"kl": 0.
|
| 372 |
"learning_rate": 3.1000000000000004e-06,
|
| 373 |
-
"loss": 0.
|
| 374 |
"step": 20,
|
| 375 |
-
"step_time": 0.
|
| 376 |
},
|
| 377 |
{
|
| 378 |
-
"clip_ratio/high_max": 0.
|
| 379 |
-
"clip_ratio/high_mean": 0.
|
| 380 |
"clip_ratio/low_mean": 0.0,
|
| 381 |
"clip_ratio/low_min": 0.0,
|
| 382 |
-
"clip_ratio/region_mean": 0.
|
| 383 |
"completions/clipped_ratio": 1.0,
|
| 384 |
"completions/max_length": 384.0,
|
| 385 |
"completions/max_terminated_length": 0.0,
|
|
@@ -387,72 +387,72 @@
|
|
| 387 |
"completions/mean_terminated_length": 0.0,
|
| 388 |
"completions/min_length": 384.0,
|
| 389 |
"completions/min_terminated_length": 0.0,
|
| 390 |
-
"entropy": 1.
|
| 391 |
"epoch": 0.875,
|
| 392 |
-
"frac_reward_zero_std": 0.
|
| 393 |
-
"grad_norm": 2.
|
| 394 |
-
"kl": 0.
|
| 395 |
"learning_rate": 3e-06,
|
| 396 |
-
"loss": -0.
|
| 397 |
"num_tokens": 151596.0,
|
| 398 |
-
"reward": -0.
|
| 399 |
-
"reward_std": 0.
|
| 400 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 401 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 402 |
"step": 21,
|
| 403 |
-
"step_time":
|
| 404 |
},
|
| 405 |
{
|
| 406 |
-
"clip_ratio/high_max": 0.
|
| 407 |
-
"clip_ratio/high_mean": 0.
|
| 408 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 409 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 410 |
-
"clip_ratio/region_mean": 0.
|
| 411 |
-
"entropy": 1.
|
| 412 |
"epoch": 0.9166666666666666,
|
| 413 |
-
"grad_norm":
|
| 414 |
-
"kl": 0.
|
| 415 |
"learning_rate": 2.9e-06,
|
| 416 |
-
"loss":
|
| 417 |
"step": 22,
|
| 418 |
-
"step_time": 0.
|
| 419 |
},
|
| 420 |
{
|
| 421 |
-
"clip_ratio/high_max": 0.
|
| 422 |
-
"clip_ratio/high_mean": 0.
|
| 423 |
-
"clip_ratio/low_mean": 0.
|
| 424 |
-
"clip_ratio/low_min": 0.
|
| 425 |
-
"clip_ratio/region_mean": 0.
|
| 426 |
-
"entropy": 1.
|
| 427 |
"epoch": 0.9583333333333334,
|
| 428 |
-
"grad_norm": 2.
|
| 429 |
-
"kl": 0.
|
| 430 |
"learning_rate": 2.8000000000000003e-06,
|
| 431 |
-
"loss": 0.
|
| 432 |
"step": 23,
|
| 433 |
-
"step_time": 0.
|
| 434 |
},
|
| 435 |
{
|
| 436 |
-
"clip_ratio/high_max": 0.
|
| 437 |
-
"clip_ratio/high_mean": 0.
|
| 438 |
-
"clip_ratio/low_mean": 0.
|
| 439 |
-
"clip_ratio/low_min": 0.
|
| 440 |
-
"clip_ratio/region_mean": 0.
|
| 441 |
-
"entropy": 0.
|
| 442 |
"epoch": 1.0,
|
| 443 |
-
"grad_norm": 1.
|
| 444 |
-
"kl": 0.
|
| 445 |
"learning_rate": 2.7000000000000004e-06,
|
| 446 |
-
"loss": 0.
|
| 447 |
"step": 24,
|
| 448 |
-
"step_time": 0.
|
| 449 |
},
|
| 450 |
{
|
| 451 |
-
"clip_ratio/high_max": 0.
|
| 452 |
-
"clip_ratio/high_mean": 0.
|
| 453 |
"clip_ratio/low_mean": 0.0,
|
| 454 |
"clip_ratio/low_min": 0.0,
|
| 455 |
-
"clip_ratio/region_mean": 0.
|
| 456 |
"completions/clipped_ratio": 1.0,
|
| 457 |
"completions/max_length": 384.0,
|
| 458 |
"completions/max_terminated_length": 0.0,
|
|
@@ -460,20 +460,20 @@
|
|
| 460 |
"completions/mean_terminated_length": 0.0,
|
| 461 |
"completions/min_length": 384.0,
|
| 462 |
"completions/min_terminated_length": 0.0,
|
| 463 |
-
"entropy": 1.
|
| 464 |
"epoch": 1.0416666666666667,
|
| 465 |
"frac_reward_zero_std": 0.0,
|
| 466 |
-
"grad_norm": 2.
|
| 467 |
-
"kl": 0.
|
| 468 |
"learning_rate": 2.6e-06,
|
| 469 |
-
"loss": 0.
|
| 470 |
"num_tokens": 177212.0,
|
| 471 |
-
"reward": -0.
|
| 472 |
-
"reward_std": 0.
|
| 473 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 474 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 475 |
"step": 25,
|
| 476 |
-
"step_time": 12.
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"clip_ratio/high_max": 0.00390625,
|
|
@@ -481,51 +481,51 @@
|
|
| 481 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 482 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 483 |
"clip_ratio/region_mean": 0.0052083334885537624,
|
| 484 |
-
"entropy":
|
| 485 |
"epoch": 1.0833333333333333,
|
| 486 |
-
"grad_norm":
|
| 487 |
-
"kl": 0.
|
| 488 |
"learning_rate": 2.5e-06,
|
| 489 |
-
"loss": -0.
|
| 490 |
"step": 26,
|
| 491 |
-
"step_time": 0.
|
| 492 |
},
|
| 493 |
{
|
| 494 |
-
"clip_ratio/high_max": 0.
|
| 495 |
-
"clip_ratio/high_mean": 0.
|
| 496 |
-
"clip_ratio/low_mean": 0.
|
| 497 |
-
"clip_ratio/low_min": 0.
|
| 498 |
-
"clip_ratio/region_mean": 0.
|
| 499 |
-
"entropy": 1.
|
| 500 |
"epoch": 1.125,
|
| 501 |
-
"grad_norm":
|
| 502 |
-
"kl": 0.
|
| 503 |
"learning_rate": 2.4000000000000003e-06,
|
| 504 |
-
"loss": -0.
|
| 505 |
"step": 27,
|
| 506 |
-
"step_time": 0.
|
| 507 |
},
|
| 508 |
{
|
| 509 |
-
"clip_ratio/high_max": 0.
|
| 510 |
-
"clip_ratio/high_mean": 0.
|
| 511 |
-
"clip_ratio/low_mean": 0.
|
| 512 |
-
"clip_ratio/low_min": 0.
|
| 513 |
-
"clip_ratio/region_mean": 0.
|
| 514 |
-
"entropy":
|
| 515 |
"epoch": 1.1666666666666667,
|
| 516 |
-
"grad_norm":
|
| 517 |
-
"kl": 0.
|
| 518 |
"learning_rate": 2.3000000000000004e-06,
|
| 519 |
-
"loss": 0.
|
| 520 |
"step": 28,
|
| 521 |
-
"step_time": 0.
|
| 522 |
},
|
| 523 |
{
|
| 524 |
-
"clip_ratio/high_max": 0.
|
| 525 |
-
"clip_ratio/high_mean": 0.
|
| 526 |
"clip_ratio/low_mean": 0.0,
|
| 527 |
"clip_ratio/low_min": 0.0,
|
| 528 |
-
"clip_ratio/region_mean": 0.
|
| 529 |
"completions/clipped_ratio": 1.0,
|
| 530 |
"completions/max_length": 384.0,
|
| 531 |
"completions/max_terminated_length": 0.0,
|
|
@@ -533,65 +533,65 @@
|
|
| 533 |
"completions/mean_terminated_length": 0.0,
|
| 534 |
"completions/min_length": 384.0,
|
| 535 |
"completions/min_terminated_length": 0.0,
|
| 536 |
-
"entropy":
|
| 537 |
"epoch": 1.2083333333333333,
|
| 538 |
"frac_reward_zero_std": 0.0,
|
| 539 |
-
"grad_norm":
|
| 540 |
-
"kl": 0.
|
| 541 |
"learning_rate": 2.2e-06,
|
| 542 |
-
"loss":
|
| 543 |
"num_tokens": 202200.0,
|
| 544 |
-
"reward": -0.
|
| 545 |
-
"reward_std": 0.
|
| 546 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 547 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 548 |
"step": 29,
|
| 549 |
-
"step_time":
|
| 550 |
},
|
| 551 |
{
|
| 552 |
-
"clip_ratio/high_max": 0.
|
| 553 |
-
"clip_ratio/high_mean": 0.
|
| 554 |
-
"clip_ratio/low_mean": 0.
|
| 555 |
-
"clip_ratio/low_min": 0.
|
| 556 |
-
"clip_ratio/region_mean": 0.
|
| 557 |
-
"entropy":
|
| 558 |
"epoch": 1.25,
|
| 559 |
-
"grad_norm": 2.
|
| 560 |
-
"kl": 0.
|
| 561 |
"learning_rate": 2.1000000000000002e-06,
|
| 562 |
-
"loss": 0.
|
| 563 |
"step": 30,
|
| 564 |
-
"step_time": 0.
|
| 565 |
},
|
| 566 |
{
|
| 567 |
-
"clip_ratio/high_max": 0.
|
| 568 |
-
"clip_ratio/high_mean": 0.
|
| 569 |
-
"clip_ratio/low_mean": 0.
|
| 570 |
-
"clip_ratio/low_min": 0.
|
| 571 |
-
"clip_ratio/region_mean": 0.
|
| 572 |
-
"entropy": 1.
|
| 573 |
"epoch": 1.2916666666666667,
|
| 574 |
-
"grad_norm": 2.
|
| 575 |
-
"kl": 0.
|
| 576 |
"learning_rate": 2.0000000000000003e-06,
|
| 577 |
-
"loss":
|
| 578 |
"step": 31,
|
| 579 |
-
"step_time": 0.
|
| 580 |
},
|
| 581 |
{
|
| 582 |
-
"clip_ratio/high_max": 0.
|
| 583 |
-
"clip_ratio/high_mean": 0.
|
| 584 |
-
"clip_ratio/low_mean": 0.
|
| 585 |
-
"clip_ratio/low_min": 0.
|
| 586 |
-
"clip_ratio/region_mean": 0.
|
| 587 |
-
"entropy": 1.
|
| 588 |
"epoch": 1.3333333333333333,
|
| 589 |
-
"grad_norm":
|
| 590 |
-
"kl": 0.
|
| 591 |
"learning_rate": 1.9000000000000002e-06,
|
| 592 |
-
"loss": -0.
|
| 593 |
"step": 32,
|
| 594 |
-
"step_time": 0.
|
| 595 |
},
|
| 596 |
{
|
| 597 |
"clip_ratio/high_max": 0.0,
|
|
@@ -606,65 +606,65 @@
|
|
| 606 |
"completions/mean_terminated_length": 0.0,
|
| 607 |
"completions/min_length": 384.0,
|
| 608 |
"completions/min_terminated_length": 0.0,
|
| 609 |
-
"entropy":
|
| 610 |
"epoch": 1.375,
|
| 611 |
"frac_reward_zero_std": 0.0,
|
| 612 |
-
"grad_norm": 2.
|
| 613 |
-
"kl": 0.
|
| 614 |
"learning_rate": 1.8000000000000001e-06,
|
| 615 |
-
"loss": 0.
|
| 616 |
"num_tokens": 227556.0,
|
| 617 |
-
"reward": -0.
|
| 618 |
-
"reward_std": 0.
|
| 619 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 620 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 621 |
"step": 33,
|
| 622 |
-
"step_time": 12.
|
| 623 |
},
|
| 624 |
{
|
| 625 |
-
"clip_ratio/high_max": 0.
|
| 626 |
-
"clip_ratio/high_mean": 0.
|
| 627 |
-
"clip_ratio/low_mean": 0.
|
| 628 |
-
"clip_ratio/low_min": 0.
|
| 629 |
-
"clip_ratio/region_mean": 0.
|
| 630 |
-
"entropy": 1.
|
| 631 |
"epoch": 1.4166666666666667,
|
| 632 |
-
"grad_norm":
|
| 633 |
-
"kl": 0.
|
| 634 |
"learning_rate": 1.7000000000000002e-06,
|
| 635 |
-
"loss": -0.
|
| 636 |
"step": 34,
|
| 637 |
-
"step_time": 0.
|
| 638 |
},
|
| 639 |
{
|
| 640 |
-
"clip_ratio/high_max": 0.
|
| 641 |
-
"clip_ratio/high_mean": 0.
|
| 642 |
-
"clip_ratio/low_mean": 0.
|
| 643 |
-
"clip_ratio/low_min": 0.
|
| 644 |
-
"clip_ratio/region_mean": 0.
|
| 645 |
-
"entropy":
|
| 646 |
"epoch": 1.4583333333333333,
|
| 647 |
-
"grad_norm": 2.
|
| 648 |
-
"kl": 0.
|
| 649 |
"learning_rate": 1.6000000000000001e-06,
|
| 650 |
-
"loss": -0.
|
| 651 |
"step": 35,
|
| 652 |
-
"step_time": 0.
|
| 653 |
},
|
| 654 |
{
|
| 655 |
-
"clip_ratio/high_max": 0.
|
| 656 |
-
"clip_ratio/high_mean": 0.
|
| 657 |
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 658 |
"clip_ratio/low_min": 0.0032552082557231188,
|
| 659 |
-
"clip_ratio/region_mean": 0.
|
| 660 |
-
"entropy":
|
| 661 |
"epoch": 1.5,
|
| 662 |
-
"grad_norm": 1.
|
| 663 |
-
"kl": 0.
|
| 664 |
"learning_rate": 1.5e-06,
|
| 665 |
-
"loss": 0.
|
| 666 |
"step": 36,
|
| 667 |
-
"step_time": 0.
|
| 668 |
},
|
| 669 |
{
|
| 670 |
"clip_ratio/high_max": 0.0,
|
|
@@ -679,72 +679,72 @@
|
|
| 679 |
"completions/mean_terminated_length": 0.0,
|
| 680 |
"completions/min_length": 384.0,
|
| 681 |
"completions/min_terminated_length": 0.0,
|
| 682 |
-
"entropy": 1.
|
| 683 |
"epoch": 1.5416666666666665,
|
| 684 |
-
"frac_reward_zero_std": 0.
|
| 685 |
-
"grad_norm":
|
| 686 |
-
"kl": 0.
|
| 687 |
"learning_rate": 1.4000000000000001e-06,
|
| 688 |
-
"loss": -0.
|
| 689 |
"num_tokens": 252640.0,
|
| 690 |
-
"reward": -0.
|
| 691 |
-
"reward_std": 0.
|
| 692 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 693 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 694 |
"step": 37,
|
| 695 |
-
"step_time":
|
| 696 |
},
|
| 697 |
{
|
| 698 |
-
"clip_ratio/high_max": 0.
|
| 699 |
-
"clip_ratio/high_mean": 0.
|
| 700 |
-
"clip_ratio/low_mean": 0.
|
| 701 |
-
"clip_ratio/low_min": 0.
|
| 702 |
-
"clip_ratio/region_mean": 0.
|
| 703 |
-
"entropy": 1.
|
| 704 |
"epoch": 1.5833333333333335,
|
| 705 |
-
"grad_norm": 1.
|
| 706 |
-
"kl": 0.
|
| 707 |
"learning_rate": 1.3e-06,
|
| 708 |
-
"loss": 0.
|
| 709 |
"step": 38,
|
| 710 |
-
"step_time": 0.
|
| 711 |
},
|
| 712 |
{
|
| 713 |
-
"clip_ratio/high_max": 0.
|
| 714 |
-
"clip_ratio/high_mean": 0.
|
| 715 |
-
"clip_ratio/low_mean": 0.
|
| 716 |
-
"clip_ratio/low_min": 0.
|
| 717 |
-
"clip_ratio/region_mean": 0.
|
| 718 |
-
"entropy": 1.
|
| 719 |
"epoch": 1.625,
|
| 720 |
-
"grad_norm":
|
| 721 |
-
"kl": 0.
|
| 722 |
"learning_rate": 1.2000000000000002e-06,
|
| 723 |
-
"loss": 0.
|
| 724 |
"step": 39,
|
| 725 |
-
"step_time": 0.
|
| 726 |
},
|
| 727 |
{
|
| 728 |
-
"clip_ratio/high_max": 0.
|
| 729 |
-
"clip_ratio/high_mean": 0.
|
| 730 |
-
"clip_ratio/low_mean": 0.
|
| 731 |
-
"clip_ratio/low_min": 0.
|
| 732 |
-
"clip_ratio/region_mean": 0.
|
| 733 |
-
"entropy":
|
| 734 |
"epoch": 1.6666666666666665,
|
| 735 |
-
"grad_norm": 2.
|
| 736 |
-
"kl": 0.
|
| 737 |
"learning_rate": 1.1e-06,
|
| 738 |
-
"loss": -0.
|
| 739 |
"step": 40,
|
| 740 |
-
"step_time": 0.
|
| 741 |
},
|
| 742 |
{
|
| 743 |
"clip_ratio/high_max": 0.0,
|
| 744 |
"clip_ratio/high_mean": 0.0,
|
| 745 |
-
"clip_ratio/low_mean": 0.
|
| 746 |
-
"clip_ratio/low_min": 0.
|
| 747 |
-
"clip_ratio/region_mean": 0.
|
| 748 |
"completions/clipped_ratio": 1.0,
|
| 749 |
"completions/max_length": 384.0,
|
| 750 |
"completions/max_terminated_length": 0.0,
|
|
@@ -752,72 +752,72 @@
|
|
| 752 |
"completions/mean_terminated_length": 0.0,
|
| 753 |
"completions/min_length": 384.0,
|
| 754 |
"completions/min_terminated_length": 0.0,
|
| 755 |
-
"entropy": 1.
|
| 756 |
"epoch": 1.7083333333333335,
|
| 757 |
-
"frac_reward_zero_std": 0.
|
| 758 |
-
"grad_norm":
|
| 759 |
-
"kl": 0.
|
| 760 |
"learning_rate": 1.0000000000000002e-06,
|
| 761 |
-
"loss": 0.
|
| 762 |
"num_tokens": 277896.0,
|
| 763 |
-
"reward": -0.
|
| 764 |
-
"reward_std": 0.
|
| 765 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 766 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 767 |
"step": 41,
|
| 768 |
-
"step_time": 12.
|
| 769 |
},
|
| 770 |
{
|
| 771 |
-
"clip_ratio/high_max": 0.
|
| 772 |
-
"clip_ratio/high_mean": 0.
|
| 773 |
-
"clip_ratio/low_mean": 0.
|
| 774 |
-
"clip_ratio/low_min": 0.
|
| 775 |
-
"clip_ratio/region_mean": 0.
|
| 776 |
-
"entropy":
|
| 777 |
"epoch": 1.75,
|
| 778 |
-
"grad_norm":
|
| 779 |
-
"kl": 0.
|
| 780 |
"learning_rate": 9.000000000000001e-07,
|
| 781 |
-
"loss": 0.
|
| 782 |
"step": 42,
|
| 783 |
-
"step_time": 0.
|
| 784 |
},
|
| 785 |
{
|
| 786 |
-
"clip_ratio/high_max": 0.
|
| 787 |
-
"clip_ratio/high_mean": 0.
|
| 788 |
-
"clip_ratio/low_mean": 0.
|
| 789 |
-
"clip_ratio/low_min": 0.
|
| 790 |
-
"clip_ratio/region_mean": 0.
|
| 791 |
-
"entropy":
|
| 792 |
"epoch": 1.7916666666666665,
|
| 793 |
-
"grad_norm":
|
| 794 |
-
"kl": 0.
|
| 795 |
"learning_rate": 8.000000000000001e-07,
|
| 796 |
-
"loss":
|
| 797 |
"step": 43,
|
| 798 |
-
"step_time": 0.
|
| 799 |
},
|
| 800 |
{
|
| 801 |
-
"clip_ratio/high_max": 0.
|
| 802 |
-
"clip_ratio/high_mean": 0.
|
| 803 |
-
"clip_ratio/low_mean": 0.
|
| 804 |
-
"clip_ratio/low_min": 0.
|
| 805 |
-
"clip_ratio/region_mean": 0.
|
| 806 |
-
"entropy": 1.
|
| 807 |
"epoch": 1.8333333333333335,
|
| 808 |
-
"grad_norm": 2.
|
| 809 |
-
"kl": 0.
|
| 810 |
"learning_rate": 7.000000000000001e-07,
|
| 811 |
-
"loss": 0.
|
| 812 |
"step": 44,
|
| 813 |
-
"step_time": 0.
|
| 814 |
},
|
| 815 |
{
|
| 816 |
-
"clip_ratio/high_max": 0.
|
| 817 |
-
"clip_ratio/high_mean": 0.
|
| 818 |
"clip_ratio/low_mean": 0.0,
|
| 819 |
"clip_ratio/low_min": 0.0,
|
| 820 |
-
"clip_ratio/region_mean": 0.
|
| 821 |
"completions/clipped_ratio": 1.0,
|
| 822 |
"completions/max_length": 384.0,
|
| 823 |
"completions/max_terminated_length": 0.0,
|
|
@@ -825,20 +825,20 @@
|
|
| 825 |
"completions/mean_terminated_length": 0.0,
|
| 826 |
"completions/min_length": 384.0,
|
| 827 |
"completions/min_terminated_length": 0.0,
|
| 828 |
-
"entropy": 1.
|
| 829 |
"epoch": 1.875,
|
| 830 |
-
"frac_reward_zero_std": 0.
|
| 831 |
-
"grad_norm":
|
| 832 |
-
"kl": 0.
|
| 833 |
"learning_rate": 6.000000000000001e-07,
|
| 834 |
-
"loss": 0.
|
| 835 |
"num_tokens": 303192.0,
|
| 836 |
-
"reward": -0.
|
| 837 |
-
"reward_std": 0.
|
| 838 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 839 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 840 |
"step": 45,
|
| 841 |
-
"step_time": 12.
|
| 842 |
},
|
| 843 |
{
|
| 844 |
"clip_ratio/high_max": 0.0026041667442768812,
|
|
@@ -846,44 +846,44 @@
|
|
| 846 |
"clip_ratio/low_mean": 0.0,
|
| 847 |
"clip_ratio/low_min": 0.0,
|
| 848 |
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 849 |
-
"entropy": 1.
|
| 850 |
"epoch": 1.9166666666666665,
|
| 851 |
-
"grad_norm": 2.
|
| 852 |
-
"kl": 0.
|
| 853 |
"learning_rate": 5.000000000000001e-07,
|
| 854 |
-
"loss": -0.
|
| 855 |
"step": 46,
|
| 856 |
-
"step_time": 0.
|
| 857 |
},
|
| 858 |
{
|
| 859 |
-
"clip_ratio/high_max": 0.
|
| 860 |
-
"clip_ratio/high_mean": 0.
|
| 861 |
-
"clip_ratio/low_mean": 0.
|
| 862 |
-
"clip_ratio/low_min": 0.
|
| 863 |
-
"clip_ratio/region_mean": 0.
|
| 864 |
-
"entropy": 1.
|
| 865 |
"epoch": 1.9583333333333335,
|
| 866 |
-
"grad_norm": 2.
|
| 867 |
-
"kl": 0.
|
| 868 |
"learning_rate": 4.0000000000000003e-07,
|
| 869 |
-
"loss": 0.
|
| 870 |
"step": 47,
|
| 871 |
-
"step_time": 0.
|
| 872 |
},
|
| 873 |
{
|
| 874 |
"clip_ratio/high_max": 0.0032552082557231188,
|
| 875 |
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 876 |
-
"clip_ratio/low_mean": 0.
|
| 877 |
-
"clip_ratio/low_min": 0.
|
| 878 |
-
"clip_ratio/region_mean": 0.
|
| 879 |
-
"entropy":
|
| 880 |
"epoch": 2.0,
|
| 881 |
-
"grad_norm": 1.
|
| 882 |
-
"kl": 0.
|
| 883 |
"learning_rate": 3.0000000000000004e-07,
|
| 884 |
-
"loss": 0.
|
| 885 |
"step": 48,
|
| 886 |
-
"step_time": 0.
|
| 887 |
},
|
| 888 |
{
|
| 889 |
"clip_ratio/high_max": 0.0,
|
|
@@ -898,35 +898,35 @@
|
|
| 898 |
"completions/mean_terminated_length": 0.0,
|
| 899 |
"completions/min_length": 384.0,
|
| 900 |
"completions/min_terminated_length": 0.0,
|
| 901 |
-
"entropy":
|
| 902 |
"epoch": 2.0416666666666665,
|
| 903 |
-
"frac_reward_zero_std": 0.
|
| 904 |
-
"grad_norm":
|
| 905 |
-
"kl": 0.
|
| 906 |
"learning_rate": 2.0000000000000002e-07,
|
| 907 |
-
"loss":
|
| 908 |
"num_tokens": 328804.0,
|
| 909 |
-
"reward": -0.
|
| 910 |
-
"reward_std": 0.
|
| 911 |
-
"rewards/GeneratorRewardFunction/mean": -0.
|
| 912 |
-
"rewards/GeneratorRewardFunction/std": 0.
|
| 913 |
"step": 49,
|
| 914 |
-
"step_time": 12.
|
| 915 |
},
|
| 916 |
{
|
| 917 |
-
"clip_ratio/high_max": 0.
|
| 918 |
-
"clip_ratio/high_mean": 0.
|
| 919 |
"clip_ratio/low_mean": 0.0,
|
| 920 |
"clip_ratio/low_min": 0.0,
|
| 921 |
-
"clip_ratio/region_mean": 0.
|
| 922 |
-
"entropy": 1.
|
| 923 |
"epoch": 2.0833333333333335,
|
| 924 |
-
"grad_norm": 2.
|
| 925 |
-
"kl": 0.
|
| 926 |
"learning_rate": 1.0000000000000001e-07,
|
| 927 |
-
"loss": -0.
|
| 928 |
"step": 50,
|
| 929 |
-
"step_time": 0.
|
| 930 |
}
|
| 931 |
],
|
| 932 |
"logging_steps": 1,
|
|
|
|
| 25 |
"entropy": 1.9362258911132812,
|
| 26 |
"epoch": 0.041666666666666664,
|
| 27 |
"frac_reward_zero_std": 0.0,
|
| 28 |
+
"grad_norm": 3.3426833152770996,
|
| 29 |
"kl": 0.0005498419050127268,
|
| 30 |
"learning_rate": 5e-06,
|
| 31 |
"loss": 0.13995857536792755,
|
|
|
|
| 35 |
"rewards/GeneratorRewardFunction/mean": -0.4352343678474426,
|
| 36 |
"rewards/GeneratorRewardFunction/std": 0.306624174118042,
|
| 37 |
"step": 1,
|
| 38 |
+
"step_time": 12.578062469000088
|
| 39 |
},
|
| 40 |
{
|
| 41 |
+
"clip_ratio/high_max": 0.00390625,
|
| 42 |
+
"clip_ratio/high_mean": 0.00390625,
|
| 43 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 44 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 45 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 46 |
+
"entropy": 1.2686206102371216,
|
| 47 |
"epoch": 0.08333333333333333,
|
| 48 |
+
"grad_norm": 2.8547239303588867,
|
| 49 |
+
"kl": 0.001546451705507934,
|
| 50 |
"learning_rate": 4.9000000000000005e-06,
|
| 51 |
+
"loss": -0.06681232899427414,
|
| 52 |
"step": 2,
|
| 53 |
+
"step_time": 0.22036709600001814
|
| 54 |
},
|
| 55 |
{
|
| 56 |
+
"clip_ratio/high_max": 0.012369791977107525,
|
| 57 |
+
"clip_ratio/high_mean": 0.012369791977107525,
|
| 58 |
+
"clip_ratio/low_mean": 0.015625,
|
| 59 |
+
"clip_ratio/low_min": 0.015625,
|
| 60 |
"clip_ratio/region_mean": 0.02799479104578495,
|
| 61 |
+
"entropy": 1.8668650388717651,
|
| 62 |
"epoch": 0.125,
|
| 63 |
+
"grad_norm": 2.4686105251312256,
|
| 64 |
+
"kl": 0.005345983896404505,
|
| 65 |
"learning_rate": 4.800000000000001e-06,
|
| 66 |
+
"loss": 0.010777520947158337,
|
| 67 |
"step": 3,
|
| 68 |
+
"step_time": 0.2199646679999887
|
| 69 |
},
|
| 70 |
{
|
| 71 |
+
"clip_ratio/high_max": 0.02083333395421505,
|
| 72 |
+
"clip_ratio/high_mean": 0.02083333395421505,
|
| 73 |
"clip_ratio/low_mean": 0.010416666977107525,
|
| 74 |
"clip_ratio/low_min": 0.010416666977107525,
|
| 75 |
+
"clip_ratio/region_mean": 0.03125,
|
| 76 |
+
"entropy": 1.1842881441116333,
|
| 77 |
"epoch": 0.16666666666666666,
|
| 78 |
+
"grad_norm": 1.569398045539856,
|
| 79 |
+
"kl": 0.0072342646308243275,
|
| 80 |
"learning_rate": 4.7e-06,
|
| 81 |
+
"loss": -0.08198019117116928,
|
| 82 |
"step": 4,
|
| 83 |
+
"step_time": 0.2201611520000597
|
| 84 |
},
|
| 85 |
{
|
| 86 |
+
"clip_ratio/high_max": 0.001953125,
|
| 87 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 88 |
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 89 |
"clip_ratio/low_min": 0.0006510416860692203,
|
| 90 |
+
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 91 |
"completions/clipped_ratio": 1.0,
|
| 92 |
"completions/max_length": 384.0,
|
| 93 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 95 |
"completions/mean_terminated_length": 0.0,
|
| 96 |
"completions/min_length": 384.0,
|
| 97 |
"completions/min_terminated_length": 0.0,
|
| 98 |
+
"entropy": 1.3128995895385742,
|
| 99 |
"epoch": 0.20833333333333334,
|
| 100 |
"frac_reward_zero_std": 0.0,
|
| 101 |
+
"grad_norm": 2.202421188354492,
|
| 102 |
+
"kl": 0.0129135362803936,
|
| 103 |
"learning_rate": 4.600000000000001e-06,
|
| 104 |
+
"loss": -0.0841849148273468,
|
| 105 |
"num_tokens": 50440.0,
|
| 106 |
+
"reward": -0.3341406285762787,
|
| 107 |
+
"reward_std": 0.3155691623687744,
|
| 108 |
+
"rewards/GeneratorRewardFunction/mean": -0.3341406285762787,
|
| 109 |
+
"rewards/GeneratorRewardFunction/std": 0.3155691623687744,
|
| 110 |
"step": 5,
|
| 111 |
+
"step_time": 12.076087126999937
|
| 112 |
},
|
| 113 |
{
|
| 114 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 115 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 116 |
+
"clip_ratio/low_mean": 0.0052083334885537624,
|
| 117 |
+
"clip_ratio/low_min": 0.0052083334885537624,
|
| 118 |
+
"clip_ratio/region_mean": 0.0078125,
|
| 119 |
+
"entropy": 1.3001914024353027,
|
| 120 |
"epoch": 0.25,
|
| 121 |
+
"grad_norm": 2.854139804840088,
|
| 122 |
+
"kl": 0.01436698716133833,
|
| 123 |
"learning_rate": 4.5e-06,
|
| 124 |
+
"loss": 0.02869725041091442,
|
| 125 |
"step": 6,
|
| 126 |
+
"step_time": 0.22715311399997518
|
| 127 |
},
|
| 128 |
{
|
| 129 |
+
"clip_ratio/high_max": 0.0071614584885537624,
|
| 130 |
+
"clip_ratio/high_mean": 0.0071614584885537624,
|
| 131 |
+
"clip_ratio/low_mean": 0.0071614584885537624,
|
| 132 |
+
"clip_ratio/low_min": 0.0071614584885537624,
|
| 133 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 134 |
+
"entropy": 1.0331100225448608,
|
| 135 |
"epoch": 0.2916666666666667,
|
| 136 |
+
"grad_norm": 1.9297211170196533,
|
| 137 |
+
"kl": 0.01791433058679104,
|
| 138 |
"learning_rate": 4.4e-06,
|
| 139 |
+
"loss": -0.028683962300419807,
|
| 140 |
"step": 7,
|
| 141 |
+
"step_time": 0.22586069900000894
|
| 142 |
},
|
| 143 |
{
|
| 144 |
"clip_ratio/high_max": 0.029296875,
|
| 145 |
"clip_ratio/high_mean": 0.029296875,
|
| 146 |
+
"clip_ratio/low_mean": 0.01171875,
|
| 147 |
+
"clip_ratio/low_min": 0.01171875,
|
| 148 |
+
"clip_ratio/region_mean": 0.041015625,
|
| 149 |
+
"entropy": 1.1462408304214478,
|
| 150 |
"epoch": 0.3333333333333333,
|
| 151 |
+
"grad_norm": 2.57124924659729,
|
| 152 |
+
"kl": 0.0388585664331913,
|
| 153 |
"learning_rate": 4.3e-06,
|
| 154 |
+
"loss": 0.08592668920755386,
|
| 155 |
"step": 8,
|
| 156 |
+
"step_time": 0.22552726200001416
|
| 157 |
},
|
| 158 |
{
|
| 159 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 168 |
"completions/mean_terminated_length": 0.0,
|
| 169 |
"completions/min_length": 384.0,
|
| 170 |
"completions/min_terminated_length": 0.0,
|
| 171 |
+
"entropy": 0.999983549118042,
|
| 172 |
"epoch": 0.375,
|
| 173 |
"frac_reward_zero_std": 0.25,
|
| 174 |
+
"grad_norm": 2.0052192211151123,
|
| 175 |
+
"kl": 0.030047910287976265,
|
| 176 |
"learning_rate": 4.2000000000000004e-06,
|
| 177 |
+
"loss": -0.05663062259554863,
|
| 178 |
"num_tokens": 75884.0,
|
| 179 |
+
"reward": -0.3902343511581421,
|
| 180 |
+
"reward_std": 0.31722894310951233,
|
| 181 |
+
"rewards/GeneratorRewardFunction/mean": -0.3902343511581421,
|
| 182 |
+
"rewards/GeneratorRewardFunction/std": 0.3172289729118347,
|
| 183 |
"step": 9,
|
| 184 |
+
"step_time": 12.047747722000054
|
| 185 |
},
|
| 186 |
{
|
| 187 |
+
"clip_ratio/high_max": 0.005859375,
|
| 188 |
+
"clip_ratio/high_mean": 0.005859375,
|
| 189 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 190 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 191 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 192 |
+
"entropy": 1.6177984476089478,
|
| 193 |
"epoch": 0.4166666666666667,
|
| 194 |
+
"grad_norm": 2.137237071990967,
|
| 195 |
+
"kl": 0.04101690649986267,
|
| 196 |
"learning_rate": 4.1e-06,
|
| 197 |
+
"loss": -0.02161034755408764,
|
| 198 |
"step": 10,
|
| 199 |
+
"step_time": 0.2252395229999138
|
| 200 |
},
|
| 201 |
{
|
| 202 |
+
"clip_ratio/high_max": 0.01692708395421505,
|
| 203 |
+
"clip_ratio/high_mean": 0.01692708395421505,
|
| 204 |
+
"clip_ratio/low_mean": 0.0065104165114462376,
|
| 205 |
+
"clip_ratio/low_min": 0.0065104165114462376,
|
| 206 |
+
"clip_ratio/region_mean": 0.0234375,
|
| 207 |
+
"entropy": 1.038699746131897,
|
| 208 |
"epoch": 0.4583333333333333,
|
| 209 |
+
"grad_norm": 2.672621965408325,
|
| 210 |
+
"kl": 0.031740155071020126,
|
| 211 |
"learning_rate": 4.000000000000001e-06,
|
| 212 |
+
"loss": 0.056199509650468826,
|
| 213 |
"step": 11,
|
| 214 |
+
"step_time": 0.22556489999999485
|
| 215 |
},
|
| 216 |
{
|
| 217 |
+
"clip_ratio/high_max": 0.01822916604578495,
|
| 218 |
+
"clip_ratio/high_mean": 0.01822916604578495,
|
| 219 |
+
"clip_ratio/low_mean": 0.010416666977107525,
|
| 220 |
+
"clip_ratio/low_min": 0.010416666977107525,
|
| 221 |
+
"clip_ratio/region_mean": 0.02864583395421505,
|
| 222 |
+
"entropy": 1.296442985534668,
|
| 223 |
"epoch": 0.5,
|
| 224 |
+
"grad_norm": 1.4488099813461304,
|
| 225 |
+
"kl": 0.04755128547549248,
|
| 226 |
"learning_rate": 3.900000000000001e-06,
|
| 227 |
+
"loss": 0.02385079860687256,
|
| 228 |
"step": 12,
|
| 229 |
+
"step_time": 0.22498397900005784
|
| 230 |
},
|
| 231 |
{
|
| 232 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 233 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 234 |
+
"clip_ratio/low_mean": 0.0,
|
| 235 |
+
"clip_ratio/low_min": 0.0,
|
| 236 |
+
"clip_ratio/region_mean": 0.0013020833721384406,
|
| 237 |
"completions/clipped_ratio": 1.0,
|
| 238 |
"completions/max_length": 384.0,
|
| 239 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 241 |
"completions/mean_terminated_length": 0.0,
|
| 242 |
"completions/min_length": 384.0,
|
| 243 |
"completions/min_terminated_length": 0.0,
|
| 244 |
+
"entropy": 1.3575109243392944,
|
| 245 |
"epoch": 0.5416666666666666,
|
| 246 |
+
"frac_reward_zero_std": 0.0,
|
| 247 |
+
"grad_norm": 2.2914443016052246,
|
| 248 |
+
"kl": 0.06257984787225723,
|
| 249 |
"learning_rate": 3.8000000000000005e-06,
|
| 250 |
+
"loss": -0.10538653284311295,
|
| 251 |
"num_tokens": 101112.0,
|
| 252 |
+
"reward": -0.22843749821186066,
|
| 253 |
+
"reward_std": 0.294514924287796,
|
| 254 |
+
"rewards/GeneratorRewardFunction/mean": -0.22843749821186066,
|
| 255 |
+
"rewards/GeneratorRewardFunction/std": 0.2945149540901184,
|
| 256 |
"step": 13,
|
| 257 |
+
"step_time": 12.01501083200003
|
| 258 |
},
|
| 259 |
{
|
| 260 |
+
"clip_ratio/high_max": 0.001953125,
|
| 261 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 262 |
+
"clip_ratio/low_mean": 0.0,
|
| 263 |
+
"clip_ratio/low_min": 0.0,
|
| 264 |
+
"clip_ratio/region_mean": 0.001953125,
|
| 265 |
+
"entropy": 1.2918612957000732,
|
| 266 |
"epoch": 0.5833333333333334,
|
| 267 |
+
"grad_norm": 3.0368542671203613,
|
| 268 |
+
"kl": 0.04979195073246956,
|
| 269 |
"learning_rate": 3.7e-06,
|
| 270 |
+
"loss": -0.003113487036898732,
|
| 271 |
"step": 14,
|
| 272 |
+
"step_time": 0.21806825399994523
|
| 273 |
},
|
| 274 |
{
|
| 275 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 276 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 277 |
+
"clip_ratio/low_mean": 0.005859375,
|
| 278 |
+
"clip_ratio/low_min": 0.005859375,
|
| 279 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 280 |
+
"entropy": 1.1081053018569946,
|
| 281 |
"epoch": 0.625,
|
| 282 |
+
"grad_norm": 3.5923683643341064,
|
| 283 |
+
"kl": 0.06817911565303802,
|
| 284 |
"learning_rate": 3.6000000000000003e-06,
|
| 285 |
+
"loss": 0.15118412673473358,
|
| 286 |
"step": 15,
|
| 287 |
+
"step_time": 0.217887520999966
|
| 288 |
},
|
| 289 |
{
|
| 290 |
+
"clip_ratio/high_max": 0.02018229104578495,
|
| 291 |
+
"clip_ratio/high_mean": 0.02018229104578495,
|
| 292 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 293 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 294 |
+
"clip_ratio/region_mean": 0.02278645895421505,
|
| 295 |
+
"entropy": 1.0803831815719604,
|
| 296 |
"epoch": 0.6666666666666666,
|
| 297 |
+
"grad_norm": 1.789110541343689,
|
| 298 |
+
"kl": 0.056480005383491516,
|
| 299 |
"learning_rate": 3.5e-06,
|
| 300 |
+
"loss": -0.03890883922576904,
|
| 301 |
"step": 16,
|
| 302 |
+
"step_time": 0.21781940799996846
|
| 303 |
},
|
| 304 |
{
|
| 305 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 314 |
"completions/mean_terminated_length": 0.0,
|
| 315 |
"completions/min_length": 384.0,
|
| 316 |
"completions/min_terminated_length": 0.0,
|
| 317 |
+
"entropy": 0.8709045052528381,
|
| 318 |
"epoch": 0.7083333333333334,
|
| 319 |
"frac_reward_zero_std": 0.0,
|
| 320 |
+
"grad_norm": 1.329393982887268,
|
| 321 |
+
"kl": 0.06073950603604317,
|
| 322 |
"learning_rate": 3.4000000000000005e-06,
|
| 323 |
+
"loss": -0.11920400708913803,
|
| 324 |
"num_tokens": 126436.0,
|
| 325 |
+
"reward": -0.2240625023841858,
|
| 326 |
+
"reward_std": 0.2881968021392822,
|
| 327 |
+
"rewards/GeneratorRewardFunction/mean": -0.2240625023841858,
|
| 328 |
+
"rewards/GeneratorRewardFunction/std": 0.2881968021392822,
|
| 329 |
"step": 17,
|
| 330 |
+
"step_time": 12.08798373600007
|
| 331 |
},
|
| 332 |
{
|
| 333 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 334 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 335 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 336 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 337 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 338 |
+
"entropy": 1.083386778831482,
|
| 339 |
"epoch": 0.75,
|
| 340 |
+
"grad_norm": 1.343295931816101,
|
| 341 |
+
"kl": 0.0919194221496582,
|
| 342 |
"learning_rate": 3.3000000000000006e-06,
|
| 343 |
+
"loss": -0.007308408617973328,
|
| 344 |
"step": 18,
|
| 345 |
+
"step_time": 0.2253850289998809
|
| 346 |
},
|
| 347 |
{
|
| 348 |
+
"clip_ratio/high_max": 0.005859375,
|
| 349 |
+
"clip_ratio/high_mean": 0.005859375,
|
| 350 |
+
"clip_ratio/low_mean": 0.0026041667442768812,
|
| 351 |
+
"clip_ratio/low_min": 0.0026041667442768812,
|
| 352 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 353 |
+
"entropy": 1.406662940979004,
|
| 354 |
"epoch": 0.7916666666666666,
|
| 355 |
+
"grad_norm": 3.3420534133911133,
|
| 356 |
+
"kl": 0.06450249999761581,
|
| 357 |
"learning_rate": 3.2000000000000003e-06,
|
| 358 |
+
"loss": 0.12472107261419296,
|
| 359 |
"step": 19,
|
| 360 |
+
"step_time": 0.22467486499999723
|
| 361 |
},
|
| 362 |
{
|
| 363 |
+
"clip_ratio/high_max": 0.011067708022892475,
|
| 364 |
+
"clip_ratio/high_mean": 0.011067708022892475,
|
| 365 |
+
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 366 |
+
"clip_ratio/low_min": 0.0032552082557231188,
|
| 367 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 368 |
+
"entropy": 1.6491953134536743,
|
| 369 |
"epoch": 0.8333333333333334,
|
| 370 |
+
"grad_norm": 3.3672103881835938,
|
| 371 |
+
"kl": 0.0773777961730957,
|
| 372 |
"learning_rate": 3.1000000000000004e-06,
|
| 373 |
+
"loss": 0.0035695277620106936,
|
| 374 |
"step": 20,
|
| 375 |
+
"step_time": 0.22389788999998927
|
| 376 |
},
|
| 377 |
{
|
| 378 |
+
"clip_ratio/high_max": 0.0006510416860692203,
|
| 379 |
+
"clip_ratio/high_mean": 0.0006510416860692203,
|
| 380 |
"clip_ratio/low_mean": 0.0,
|
| 381 |
"clip_ratio/low_min": 0.0,
|
| 382 |
+
"clip_ratio/region_mean": 0.0006510416860692203,
|
| 383 |
"completions/clipped_ratio": 1.0,
|
| 384 |
"completions/max_length": 384.0,
|
| 385 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 387 |
"completions/mean_terminated_length": 0.0,
|
| 388 |
"completions/min_length": 384.0,
|
| 389 |
"completions/min_terminated_length": 0.0,
|
| 390 |
+
"entropy": 1.4178005456924438,
|
| 391 |
"epoch": 0.875,
|
| 392 |
+
"frac_reward_zero_std": 0.0,
|
| 393 |
+
"grad_norm": 2.6333799362182617,
|
| 394 |
+
"kl": 0.07392226904630661,
|
| 395 |
"learning_rate": 3e-06,
|
| 396 |
+
"loss": -0.09101495891809464,
|
| 397 |
"num_tokens": 151596.0,
|
| 398 |
+
"reward": -0.15257811546325684,
|
| 399 |
+
"reward_std": 0.24854345619678497,
|
| 400 |
+
"rewards/GeneratorRewardFunction/mean": -0.15257811546325684,
|
| 401 |
+
"rewards/GeneratorRewardFunction/std": 0.24854345619678497,
|
| 402 |
"step": 21,
|
| 403 |
+
"step_time": 12.020297105999816
|
| 404 |
},
|
| 405 |
{
|
| 406 |
+
"clip_ratio/high_max": 0.0006510416860692203,
|
| 407 |
+
"clip_ratio/high_mean": 0.0006510416860692203,
|
| 408 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 409 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 410 |
+
"clip_ratio/region_mean": 0.001953125,
|
| 411 |
+
"entropy": 1.2036248445510864,
|
| 412 |
"epoch": 0.9166666666666666,
|
| 413 |
+
"grad_norm": 2.1499149799346924,
|
| 414 |
+
"kl": 0.0772874653339386,
|
| 415 |
"learning_rate": 2.9e-06,
|
| 416 |
+
"loss": 0.08120749890804291,
|
| 417 |
"step": 22,
|
| 418 |
+
"step_time": 0.21995178900010615
|
| 419 |
},
|
| 420 |
{
|
| 421 |
+
"clip_ratio/high_max": 0.0032552082557231188,
|
| 422 |
+
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 423 |
+
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 424 |
+
"clip_ratio/low_min": 0.0032552082557231188,
|
| 425 |
+
"clip_ratio/region_mean": 0.0065104165114462376,
|
| 426 |
+
"entropy": 1.1966055631637573,
|
| 427 |
"epoch": 0.9583333333333334,
|
| 428 |
+
"grad_norm": 2.0064616203308105,
|
| 429 |
+
"kl": 0.07331382483243942,
|
| 430 |
"learning_rate": 2.8000000000000003e-06,
|
| 431 |
+
"loss": 0.03140506148338318,
|
| 432 |
"step": 23,
|
| 433 |
+
"step_time": 0.21996421700009705
|
| 434 |
},
|
| 435 |
{
|
| 436 |
+
"clip_ratio/high_max": 0.011067708022892475,
|
| 437 |
+
"clip_ratio/high_mean": 0.011067708022892475,
|
| 438 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 439 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 440 |
+
"clip_ratio/region_mean": 0.01171875,
|
| 441 |
+
"entropy": 0.9102082252502441,
|
| 442 |
"epoch": 1.0,
|
| 443 |
+
"grad_norm": 1.7175334692001343,
|
| 444 |
+
"kl": 0.14611481130123138,
|
| 445 |
"learning_rate": 2.7000000000000004e-06,
|
| 446 |
+
"loss": -0.021010393276810646,
|
| 447 |
"step": 24,
|
| 448 |
+
"step_time": 0.21931950600014716
|
| 449 |
},
|
| 450 |
{
|
| 451 |
+
"clip_ratio/high_max": 0.0,
|
| 452 |
+
"clip_ratio/high_mean": 0.0,
|
| 453 |
"clip_ratio/low_mean": 0.0,
|
| 454 |
"clip_ratio/low_min": 0.0,
|
| 455 |
+
"clip_ratio/region_mean": 0.0,
|
| 456 |
"completions/clipped_ratio": 1.0,
|
| 457 |
"completions/max_length": 384.0,
|
| 458 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 460 |
"completions/mean_terminated_length": 0.0,
|
| 461 |
"completions/min_length": 384.0,
|
| 462 |
"completions/min_terminated_length": 0.0,
|
| 463 |
+
"entropy": 1.9153881072998047,
|
| 464 |
"epoch": 1.0416666666666667,
|
| 465 |
"frac_reward_zero_std": 0.0,
|
| 466 |
+
"grad_norm": 2.3460445404052734,
|
| 467 |
+
"kl": 0.09710023552179337,
|
| 468 |
"learning_rate": 2.6e-06,
|
| 469 |
+
"loss": 0.015220091678202152,
|
| 470 |
"num_tokens": 177212.0,
|
| 471 |
+
"reward": -0.15656250715255737,
|
| 472 |
+
"reward_std": 0.22349287569522858,
|
| 473 |
+
"rewards/GeneratorRewardFunction/mean": -0.15656250715255737,
|
| 474 |
+
"rewards/GeneratorRewardFunction/std": 0.22349286079406738,
|
| 475 |
"step": 25,
|
| 476 |
+
"step_time": 12.153388549000056
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"clip_ratio/high_max": 0.00390625,
|
|
|
|
| 481 |
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 482 |
"clip_ratio/low_min": 0.0013020833721384406,
|
| 483 |
"clip_ratio/region_mean": 0.0052083334885537624,
|
| 484 |
+
"entropy": 1.365325927734375,
|
| 485 |
"epoch": 1.0833333333333333,
|
| 486 |
+
"grad_norm": 1.8710312843322754,
|
| 487 |
+
"kl": 0.0985046848654747,
|
| 488 |
"learning_rate": 2.5e-06,
|
| 489 |
+
"loss": -0.02838735282421112,
|
| 490 |
"step": 26,
|
| 491 |
+
"step_time": 0.22659933299996737
|
| 492 |
},
|
| 493 |
{
|
| 494 |
+
"clip_ratio/high_max": 0.008463541977107525,
|
| 495 |
+
"clip_ratio/high_mean": 0.008463541977107525,
|
| 496 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 497 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 498 |
+
"clip_ratio/region_mean": 0.009765625,
|
| 499 |
+
"entropy": 1.2517439126968384,
|
| 500 |
"epoch": 1.125,
|
| 501 |
+
"grad_norm": 2.821958303451538,
|
| 502 |
+
"kl": 0.09274079650640488,
|
| 503 |
"learning_rate": 2.4000000000000003e-06,
|
| 504 |
+
"loss": -0.007298170123249292,
|
| 505 |
"step": 27,
|
| 506 |
+
"step_time": 0.22647249999999985
|
| 507 |
},
|
| 508 |
{
|
| 509 |
+
"clip_ratio/high_max": 0.0052083334885537624,
|
| 510 |
+
"clip_ratio/high_mean": 0.0052083334885537624,
|
| 511 |
+
"clip_ratio/low_mean": 0.009114583022892475,
|
| 512 |
+
"clip_ratio/low_min": 0.009114583022892475,
|
| 513 |
+
"clip_ratio/region_mean": 0.014322916977107525,
|
| 514 |
+
"entropy": 2.0579044818878174,
|
| 515 |
"epoch": 1.1666666666666667,
|
| 516 |
+
"grad_norm": 3.259742259979248,
|
| 517 |
+
"kl": 0.10746321082115173,
|
| 518 |
"learning_rate": 2.3000000000000004e-06,
|
| 519 |
+
"loss": 0.021702758967876434,
|
| 520 |
"step": 28,
|
| 521 |
+
"step_time": 0.22640677999993386
|
| 522 |
},
|
| 523 |
{
|
| 524 |
+
"clip_ratio/high_max": 0.0,
|
| 525 |
+
"clip_ratio/high_mean": 0.0,
|
| 526 |
"clip_ratio/low_mean": 0.0,
|
| 527 |
"clip_ratio/low_min": 0.0,
|
| 528 |
+
"clip_ratio/region_mean": 0.0,
|
| 529 |
"completions/clipped_ratio": 1.0,
|
| 530 |
"completions/max_length": 384.0,
|
| 531 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 533 |
"completions/mean_terminated_length": 0.0,
|
| 534 |
"completions/min_length": 384.0,
|
| 535 |
"completions/min_terminated_length": 0.0,
|
| 536 |
+
"entropy": 1.3725861310958862,
|
| 537 |
"epoch": 1.2083333333333333,
|
| 538 |
"frac_reward_zero_std": 0.0,
|
| 539 |
+
"grad_norm": 1.8806989192962646,
|
| 540 |
+
"kl": 0.11961983889341354,
|
| 541 |
"learning_rate": 2.2e-06,
|
| 542 |
+
"loss": 0.07187109440565109,
|
| 543 |
"num_tokens": 202200.0,
|
| 544 |
+
"reward": -0.0561029389500618,
|
| 545 |
+
"reward_std": 0.314301997423172,
|
| 546 |
+
"rewards/GeneratorRewardFunction/mean": -0.0561029389500618,
|
| 547 |
+
"rewards/GeneratorRewardFunction/std": 0.314301997423172,
|
| 548 |
"step": 29,
|
| 549 |
+
"step_time": 13.662849896999887
|
| 550 |
},
|
| 551 |
{
|
| 552 |
+
"clip_ratio/high_max": 0.001953125,
|
| 553 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 554 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 555 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 556 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 557 |
+
"entropy": 1.2213298082351685,
|
| 558 |
"epoch": 1.25,
|
| 559 |
+
"grad_norm": 2.1918396949768066,
|
| 560 |
+
"kl": 0.12398240715265274,
|
| 561 |
"learning_rate": 2.1000000000000002e-06,
|
| 562 |
+
"loss": -0.052896980196237564,
|
| 563 |
"step": 30,
|
| 564 |
+
"step_time": 0.2210835590001352
|
| 565 |
},
|
| 566 |
{
|
| 567 |
+
"clip_ratio/high_max": 0.001953125,
|
| 568 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 569 |
+
"clip_ratio/low_mean": 0.001953125,
|
| 570 |
+
"clip_ratio/low_min": 0.001953125,
|
| 571 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 572 |
+
"entropy": 1.2683231830596924,
|
| 573 |
"epoch": 1.2916666666666667,
|
| 574 |
+
"grad_norm": 2.524726390838623,
|
| 575 |
+
"kl": 0.14297537505626678,
|
| 576 |
"learning_rate": 2.0000000000000003e-06,
|
| 577 |
+
"loss": 0.12745414674282074,
|
| 578 |
"step": 31,
|
| 579 |
+
"step_time": 0.22096665699996265
|
| 580 |
},
|
| 581 |
{
|
| 582 |
+
"clip_ratio/high_max": 0.0032552082557231188,
|
| 583 |
+
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 584 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 585 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 586 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 587 |
+
"entropy": 1.0583091974258423,
|
| 588 |
"epoch": 1.3333333333333333,
|
| 589 |
+
"grad_norm": 2.408073902130127,
|
| 590 |
+
"kl": 0.0881701335310936,
|
| 591 |
"learning_rate": 1.9000000000000002e-06,
|
| 592 |
+
"loss": -0.14430458843708038,
|
| 593 |
"step": 32,
|
| 594 |
+
"step_time": 0.21999255600007928
|
| 595 |
},
|
| 596 |
{
|
| 597 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 606 |
"completions/mean_terminated_length": 0.0,
|
| 607 |
"completions/min_length": 384.0,
|
| 608 |
"completions/min_terminated_length": 0.0,
|
| 609 |
+
"entropy": 1.3751106262207031,
|
| 610 |
"epoch": 1.375,
|
| 611 |
"frac_reward_zero_std": 0.0,
|
| 612 |
+
"grad_norm": 2.4002864360809326,
|
| 613 |
+
"kl": 0.0900505781173706,
|
| 614 |
"learning_rate": 1.8000000000000001e-06,
|
| 615 |
+
"loss": 0.06270528584718704,
|
| 616 |
"num_tokens": 227556.0,
|
| 617 |
+
"reward": -0.11414062231779099,
|
| 618 |
+
"reward_std": 0.21683935821056366,
|
| 619 |
+
"rewards/GeneratorRewardFunction/mean": -0.11414062231779099,
|
| 620 |
+
"rewards/GeneratorRewardFunction/std": 0.21683938801288605,
|
| 621 |
"step": 33,
|
| 622 |
+
"step_time": 12.070902493999938
|
| 623 |
},
|
| 624 |
{
|
| 625 |
+
"clip_ratio/high_max": 0.0045572915114462376,
|
| 626 |
+
"clip_ratio/high_mean": 0.0045572915114462376,
|
| 627 |
+
"clip_ratio/low_mean": 0.0,
|
| 628 |
+
"clip_ratio/low_min": 0.0,
|
| 629 |
+
"clip_ratio/region_mean": 0.0045572915114462376,
|
| 630 |
+
"entropy": 1.2606719732284546,
|
| 631 |
"epoch": 1.4166666666666667,
|
| 632 |
+
"grad_norm": 1.671729326248169,
|
| 633 |
+
"kl": 0.1210540160536766,
|
| 634 |
"learning_rate": 1.7000000000000002e-06,
|
| 635 |
+
"loss": -0.04401962831616402,
|
| 636 |
"step": 34,
|
| 637 |
+
"step_time": 0.2276347459999215
|
| 638 |
},
|
| 639 |
{
|
| 640 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 641 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 642 |
+
"clip_ratio/low_mean": 0.001953125,
|
| 643 |
+
"clip_ratio/low_min": 0.001953125,
|
| 644 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 645 |
+
"entropy": 1.2780500650405884,
|
| 646 |
"epoch": 1.4583333333333333,
|
| 647 |
+
"grad_norm": 2.278010845184326,
|
| 648 |
+
"kl": 0.11484409123659134,
|
| 649 |
"learning_rate": 1.6000000000000001e-06,
|
| 650 |
+
"loss": -0.08475238084793091,
|
| 651 |
"step": 35,
|
| 652 |
+
"step_time": 0.22882699100000536
|
| 653 |
},
|
| 654 |
{
|
| 655 |
+
"clip_ratio/high_max": 0.0052083334885537624,
|
| 656 |
+
"clip_ratio/high_mean": 0.0052083334885537624,
|
| 657 |
"clip_ratio/low_mean": 0.0032552082557231188,
|
| 658 |
"clip_ratio/low_min": 0.0032552082557231188,
|
| 659 |
+
"clip_ratio/region_mean": 0.008463541977107525,
|
| 660 |
+
"entropy": 1.0553101301193237,
|
| 661 |
"epoch": 1.5,
|
| 662 |
+
"grad_norm": 1.582037091255188,
|
| 663 |
+
"kl": 0.12029703706502914,
|
| 664 |
"learning_rate": 1.5e-06,
|
| 665 |
+
"loss": 0.06627888232469559,
|
| 666 |
"step": 36,
|
| 667 |
+
"step_time": 0.22751581399984389
|
| 668 |
},
|
| 669 |
{
|
| 670 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 679 |
"completions/mean_terminated_length": 0.0,
|
| 680 |
"completions/min_length": 384.0,
|
| 681 |
"completions/min_terminated_length": 0.0,
|
| 682 |
+
"entropy": 1.0647958517074585,
|
| 683 |
"epoch": 1.5416666666666665,
|
| 684 |
+
"frac_reward_zero_std": 0.0,
|
| 685 |
+
"grad_norm": 2.1763558387756348,
|
| 686 |
+
"kl": 0.08708903193473816,
|
| 687 |
"learning_rate": 1.4000000000000001e-06,
|
| 688 |
+
"loss": -0.00017260713502764702,
|
| 689 |
"num_tokens": 252640.0,
|
| 690 |
+
"reward": -0.10210937261581421,
|
| 691 |
+
"reward_std": 0.19573244452476501,
|
| 692 |
+
"rewards/GeneratorRewardFunction/mean": -0.10210937261581421,
|
| 693 |
+
"rewards/GeneratorRewardFunction/std": 0.1957324594259262,
|
| 694 |
"step": 37,
|
| 695 |
+
"step_time": 12.015305628000078
|
| 696 |
},
|
| 697 |
{
|
| 698 |
+
"clip_ratio/high_max": 0.0026041667442768812,
|
| 699 |
+
"clip_ratio/high_mean": 0.0026041667442768812,
|
| 700 |
+
"clip_ratio/low_mean": 0.0,
|
| 701 |
+
"clip_ratio/low_min": 0.0,
|
| 702 |
+
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 703 |
+
"entropy": 1.0041130781173706,
|
| 704 |
"epoch": 1.5833333333333335,
|
| 705 |
+
"grad_norm": 1.6093602180480957,
|
| 706 |
+
"kl": 0.11537543684244156,
|
| 707 |
"learning_rate": 1.3e-06,
|
| 708 |
+
"loss": -0.12453166395425797,
|
| 709 |
"step": 38,
|
| 710 |
+
"step_time": 0.22048816200003785
|
| 711 |
},
|
| 712 |
{
|
| 713 |
+
"clip_ratio/high_max": 0.0045572915114462376,
|
| 714 |
+
"clip_ratio/high_mean": 0.0045572915114462376,
|
| 715 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 716 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 717 |
+
"clip_ratio/region_mean": 0.005859375,
|
| 718 |
+
"entropy": 1.500306487083435,
|
| 719 |
"epoch": 1.625,
|
| 720 |
+
"grad_norm": 3.409069299697876,
|
| 721 |
+
"kl": 0.10904627293348312,
|
| 722 |
"learning_rate": 1.2000000000000002e-06,
|
| 723 |
+
"loss": 0.12661518156528473,
|
| 724 |
"step": 39,
|
| 725 |
+
"step_time": 0.22087437000004684
|
| 726 |
},
|
| 727 |
{
|
| 728 |
+
"clip_ratio/high_max": 0.0078125,
|
| 729 |
+
"clip_ratio/high_mean": 0.0078125,
|
| 730 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 731 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 732 |
+
"clip_ratio/region_mean": 0.009114583022892475,
|
| 733 |
+
"entropy": 1.0560635328292847,
|
| 734 |
"epoch": 1.6666666666666665,
|
| 735 |
+
"grad_norm": 2.0718417167663574,
|
| 736 |
+
"kl": 0.11926760524511337,
|
| 737 |
"learning_rate": 1.1e-06,
|
| 738 |
+
"loss": -0.0004449083062354475,
|
| 739 |
"step": 40,
|
| 740 |
+
"step_time": 0.2202887500000088
|
| 741 |
},
|
| 742 |
{
|
| 743 |
"clip_ratio/high_max": 0.0,
|
| 744 |
"clip_ratio/high_mean": 0.0,
|
| 745 |
+
"clip_ratio/low_mean": 0.0013020833721384406,
|
| 746 |
+
"clip_ratio/low_min": 0.0013020833721384406,
|
| 747 |
+
"clip_ratio/region_mean": 0.0013020833721384406,
|
| 748 |
"completions/clipped_ratio": 1.0,
|
| 749 |
"completions/max_length": 384.0,
|
| 750 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 752 |
"completions/mean_terminated_length": 0.0,
|
| 753 |
"completions/min_length": 384.0,
|
| 754 |
"completions/min_terminated_length": 0.0,
|
| 755 |
+
"entropy": 1.0184931755065918,
|
| 756 |
"epoch": 1.7083333333333335,
|
| 757 |
+
"frac_reward_zero_std": 0.0,
|
| 758 |
+
"grad_norm": 1.9755194187164307,
|
| 759 |
+
"kl": 0.1180298700928688,
|
| 760 |
"learning_rate": 1.0000000000000002e-06,
|
| 761 |
+
"loss": 0.03202051296830177,
|
| 762 |
"num_tokens": 277896.0,
|
| 763 |
+
"reward": -0.06937499344348907,
|
| 764 |
+
"reward_std": 0.1560969203710556,
|
| 765 |
+
"rewards/GeneratorRewardFunction/mean": -0.06937499344348907,
|
| 766 |
+
"rewards/GeneratorRewardFunction/std": 0.1560969203710556,
|
| 767 |
"step": 41,
|
| 768 |
+
"step_time": 12.091393732999904
|
| 769 |
},
|
| 770 |
{
|
| 771 |
+
"clip_ratio/high_max": 0.0032552082557231188,
|
| 772 |
+
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 773 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 774 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 775 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 776 |
+
"entropy": 0.8101570010185242,
|
| 777 |
"epoch": 1.75,
|
| 778 |
+
"grad_norm": 2.101008653640747,
|
| 779 |
+
"kl": 0.13180766999721527,
|
| 780 |
"learning_rate": 9.000000000000001e-07,
|
| 781 |
+
"loss": -0.03199642524123192,
|
| 782 |
"step": 42,
|
| 783 |
+
"step_time": 0.22810021400005098
|
| 784 |
},
|
| 785 |
{
|
| 786 |
+
"clip_ratio/high_max": 0.001953125,
|
| 787 |
+
"clip_ratio/high_mean": 0.001953125,
|
| 788 |
+
"clip_ratio/low_mean": 0.0006510416860692203,
|
| 789 |
+
"clip_ratio/low_min": 0.0006510416860692203,
|
| 790 |
+
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 791 |
+
"entropy": 0.9268913269042969,
|
| 792 |
"epoch": 1.7916666666666665,
|
| 793 |
+
"grad_norm": 2.1574151515960693,
|
| 794 |
+
"kl": 0.11732880026102066,
|
| 795 |
"learning_rate": 8.000000000000001e-07,
|
| 796 |
+
"loss": 0.0002514577645342797,
|
| 797 |
"step": 43,
|
| 798 |
+
"step_time": 0.22811048399989886
|
| 799 |
},
|
| 800 |
{
|
| 801 |
+
"clip_ratio/high_max": 0.00390625,
|
| 802 |
+
"clip_ratio/high_mean": 0.00390625,
|
| 803 |
+
"clip_ratio/low_mean": 0.0,
|
| 804 |
+
"clip_ratio/low_min": 0.0,
|
| 805 |
+
"clip_ratio/region_mean": 0.00390625,
|
| 806 |
+
"entropy": 1.145074486732483,
|
| 807 |
"epoch": 1.8333333333333335,
|
| 808 |
+
"grad_norm": 2.5536458492279053,
|
| 809 |
+
"kl": 0.12928128242492676,
|
| 810 |
"learning_rate": 7.000000000000001e-07,
|
| 811 |
+
"loss": 0.0016053098952397704,
|
| 812 |
"step": 44,
|
| 813 |
+
"step_time": 0.22788419799985604
|
| 814 |
},
|
| 815 |
{
|
| 816 |
+
"clip_ratio/high_max": 0.0006510416860692203,
|
| 817 |
+
"clip_ratio/high_mean": 0.0006510416860692203,
|
| 818 |
"clip_ratio/low_mean": 0.0,
|
| 819 |
"clip_ratio/low_min": 0.0,
|
| 820 |
+
"clip_ratio/region_mean": 0.0006510416860692203,
|
| 821 |
"completions/clipped_ratio": 1.0,
|
| 822 |
"completions/max_length": 384.0,
|
| 823 |
"completions/max_terminated_length": 0.0,
|
|
|
|
| 825 |
"completions/mean_terminated_length": 0.0,
|
| 826 |
"completions/min_length": 384.0,
|
| 827 |
"completions/min_terminated_length": 0.0,
|
| 828 |
+
"entropy": 1.1803818941116333,
|
| 829 |
"epoch": 1.875,
|
| 830 |
+
"frac_reward_zero_std": 0.0,
|
| 831 |
+
"grad_norm": 3.2263009548187256,
|
| 832 |
+
"kl": 0.13209813833236694,
|
| 833 |
"learning_rate": 6.000000000000001e-07,
|
| 834 |
+
"loss": 0.1281612068414688,
|
| 835 |
"num_tokens": 303192.0,
|
| 836 |
+
"reward": -0.11374999582767487,
|
| 837 |
+
"reward_std": 0.18029142916202545,
|
| 838 |
+
"rewards/GeneratorRewardFunction/mean": -0.11374999582767487,
|
| 839 |
+
"rewards/GeneratorRewardFunction/std": 0.18029142916202545,
|
| 840 |
"step": 45,
|
| 841 |
+
"step_time": 12.014625670999976
|
| 842 |
},
|
| 843 |
{
|
| 844 |
"clip_ratio/high_max": 0.0026041667442768812,
|
|
|
|
| 846 |
"clip_ratio/low_mean": 0.0,
|
| 847 |
"clip_ratio/low_min": 0.0,
|
| 848 |
"clip_ratio/region_mean": 0.0026041667442768812,
|
| 849 |
+
"entropy": 1.6430233716964722,
|
| 850 |
"epoch": 1.9166666666666665,
|
| 851 |
+
"grad_norm": 2.463127851486206,
|
| 852 |
+
"kl": 0.11944004148244858,
|
| 853 |
"learning_rate": 5.000000000000001e-07,
|
| 854 |
+
"loss": -0.01078779250383377,
|
| 855 |
"step": 46,
|
| 856 |
+
"step_time": 0.22117237800011935
|
| 857 |
},
|
| 858 |
{
|
| 859 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 860 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 861 |
+
"clip_ratio/low_mean": 0.0,
|
| 862 |
+
"clip_ratio/low_min": 0.0,
|
| 863 |
+
"clip_ratio/region_mean": 0.0013020833721384406,
|
| 864 |
+
"entropy": 1.1240859031677246,
|
| 865 |
"epoch": 1.9583333333333335,
|
| 866 |
+
"grad_norm": 2.1054372787475586,
|
| 867 |
+
"kl": 0.13911886513233185,
|
| 868 |
"learning_rate": 4.0000000000000003e-07,
|
| 869 |
+
"loss": 0.001417159684933722,
|
| 870 |
"step": 47,
|
| 871 |
+
"step_time": 0.2201927370001613
|
| 872 |
},
|
| 873 |
{
|
| 874 |
"clip_ratio/high_max": 0.0032552082557231188,
|
| 875 |
"clip_ratio/high_mean": 0.0032552082557231188,
|
| 876 |
+
"clip_ratio/low_mean": 0.0,
|
| 877 |
+
"clip_ratio/low_min": 0.0,
|
| 878 |
+
"clip_ratio/region_mean": 0.0032552082557231188,
|
| 879 |
+
"entropy": 1.3605166673660278,
|
| 880 |
"epoch": 2.0,
|
| 881 |
+
"grad_norm": 1.7440528869628906,
|
| 882 |
+
"kl": 0.14588220417499542,
|
| 883 |
"learning_rate": 3.0000000000000004e-07,
|
| 884 |
+
"loss": -0.11717051267623901,
|
| 885 |
"step": 48,
|
| 886 |
+
"step_time": 0.21969574700005978
|
| 887 |
},
|
| 888 |
{
|
| 889 |
"clip_ratio/high_max": 0.0,
|
|
|
|
| 898 |
"completions/mean_terminated_length": 0.0,
|
| 899 |
"completions/min_length": 384.0,
|
| 900 |
"completions/min_terminated_length": 0.0,
|
| 901 |
+
"entropy": 0.9781540036201477,
|
| 902 |
"epoch": 2.0416666666666665,
|
| 903 |
+
"frac_reward_zero_std": 0.0,
|
| 904 |
+
"grad_norm": 2.4057631492614746,
|
| 905 |
+
"kl": 0.14009785652160645,
|
| 906 |
"learning_rate": 2.0000000000000002e-07,
|
| 907 |
+
"loss": 0.06281977146863937,
|
| 908 |
"num_tokens": 328804.0,
|
| 909 |
+
"reward": -0.07187499850988388,
|
| 910 |
+
"reward_std": 0.11617336422204971,
|
| 911 |
+
"rewards/GeneratorRewardFunction/mean": -0.07187499850988388,
|
| 912 |
+
"rewards/GeneratorRewardFunction/std": 0.11617336422204971,
|
| 913 |
"step": 49,
|
| 914 |
+
"step_time": 12.04901073699989
|
| 915 |
},
|
| 916 |
{
|
| 917 |
+
"clip_ratio/high_max": 0.0013020833721384406,
|
| 918 |
+
"clip_ratio/high_mean": 0.0013020833721384406,
|
| 919 |
"clip_ratio/low_mean": 0.0,
|
| 920 |
"clip_ratio/low_min": 0.0,
|
| 921 |
+
"clip_ratio/region_mean": 0.0013020833721384406,
|
| 922 |
+
"entropy": 1.6572185754776,
|
| 923 |
"epoch": 2.0833333333333335,
|
| 924 |
+
"grad_norm": 2.6693296432495117,
|
| 925 |
+
"kl": 0.13599954545497894,
|
| 926 |
"learning_rate": 1.0000000000000001e-07,
|
| 927 |
+
"loss": -0.16521048545837402,
|
| 928 |
"step": 50,
|
| 929 |
+
"step_time": 0.23019724899995708
|
| 930 |
}
|
| 931 |
],
|
| 932 |
"logging_steps": 1,
|
self_play_hf_a10g_train/round_001/generator_train/final_model/model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1976163472
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6fa4eed67a84ce4076ba3848a078496971cd34ba048c794e52cc3b4aab54a27
|
| 3 |
size 1976163472
|