diff --git a/README.md b/README.md
index ebb2907a5bd875891394af6dd359f4bde10dad33..732072c60b0c35579b98cadfe406bf0eaf4ee376 100644
--- a/README.md
+++ b/README.md
@@ -1,21 +1,20 @@
---
-base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
+base_model: unsloth/Qwen2.5-1.5B-Instruct
library_name: peft
model_name: grpo_checkpoint
tags:
-- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
- grpo
- lora
- transformers
- trl
-- unsloth
licence: license
pipeline_tag: text-generation
---
# Model Card for grpo_checkpoint
-This model is a fine-tuned version of [unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit).
+This model is a fine-tuned version of [unsloth/Qwen2.5-1.5B-Instruct](https://huggingface.co/unsloth/Qwen2.5-1.5B-Instruct).
It has been trained using [TRL](https://github.com/huggingface/trl).
## Quick start
@@ -34,15 +33,16 @@ print(output["generated_text"])
+
This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
### Framework versions
-- PEFT 0.18.1
-- TRL: 0.24.0
-- Transformers: 5.5.0
-- Pytorch: 2.10.0+cu128
-- Datasets: 4.3.0
+- PEFT 0.19.1
+- TRL: 1.3.0
+- Transformers: 5.7.0
+- Pytorch: 2.6.0+cu124
+- Datasets: 4.8.5
- Tokenizers: 0.22.2
## Citations
@@ -56,18 +56,16 @@ Cite GRPO as:
year = 2024,
eprint = {arXiv:2402.03300},
}
-
```
Cite TRL as:
```bibtex
-@misc{vonwerra2022trl,
- title = {{TRL: Transformer Reinforcement Learning}},
- author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
- year = 2020,
- journal = {GitHub repository},
- publisher = {GitHub},
- howpublished = {\url{https://github.com/huggingface/trl}}
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
}
```
\ No newline at end of file
diff --git a/adapter_config.json b/adapter_config.json
index b79ee98cab9fe9bafd1f7a7f8f257f268bb548e8..1335f363e822e5c942fa1ace6e0eb5fabdd99524 100644
--- a/adapter_config.json
+++ b/adapter_config.json
@@ -2,12 +2,8 @@
"alora_invocation_tokens": null,
"alpha_pattern": {},
"arrow_config": null,
- "auto_mapping": {
- "base_model_class": "Qwen2ForCausalLM",
- "parent_library": "transformers.models.qwen2.modeling_qwen2",
- "unsloth_fixed": true
- },
- "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
"bias": "none",
"corda_config": null,
"ensure_weight_tying": false,
@@ -23,27 +19,24 @@
"lora_alpha": 16,
"lora_bias": false,
"lora_dropout": 0.05,
+ "lora_ga_config": null,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": null,
"peft_type": "LORA",
- "peft_version": "0.18.1",
+ "peft_version": "0.19.1",
"qalora_group_size": 16,
"r": 16,
"rank_pattern": {},
"revision": null,
"target_modules": [
- "up_proj",
"q_proj",
- "gate_proj",
- "down_proj",
- "v_proj",
- "o_proj",
- "k_proj"
+ "v_proj"
],
"target_parameters": null,
"task_type": "CAUSAL_LM",
"trainable_token_indices": null,
+ "use_bdlora": null,
"use_dora": false,
"use_qalora": false,
"use_rslora": false
diff --git a/adapter_model.safetensors b/adapter_model.safetensors
index 2e1b52f997c2be491dc20c5d1cc32010e4020170..91858393ce9c2d5cb7d204530d11113af62ab24d 100644
--- a/adapter_model.safetensors
+++ b/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:9b16b3a8ae1458fdc5aab7ecb05087132f16ef915d6a57271be80187a0e4760e
-size 73911112
+oid sha256:37d50e9249dbc53f60027557052a860b27022847dd92972fcb41cbf4f17869a3
+size 8731128
diff --git a/checkpoint-100/README.md b/checkpoint-100/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-100/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-100/adapter_config.json b/checkpoint-100/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-100/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-100/adapter_model.safetensors b/checkpoint-100/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb46a185ca253ce73eaf9de0a5a55e368a1c8b
--- /dev/null
+++ b/checkpoint-100/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b765e914dcf9c550a8e56ee013a86678674612688bd65ab80410d7b255546e2f
+size 8731128
diff --git a/checkpoint-100/chat_template.jinja b/checkpoint-100/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-100/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-100/optimizer.pt b/checkpoint-100/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2529ee600e59591f8284011977f830f44af25228
--- /dev/null
+++ b/checkpoint-100/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a12e35a712271cd52af220515dc54813cb15987e84f037d958d20cd6cee68fff
+size 17526842
diff --git a/checkpoint-100/ref/adapter_config.json b/checkpoint-100/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-100/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-100/ref/adapter_model.safetensors b/checkpoint-100/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-100/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-100/rng_state.pth b/checkpoint-100/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1f81d453bd6b568764b658f5df2c4f2655db5d58
--- /dev/null
+++ b/checkpoint-100/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ab604215494ab766d54b5d034814a38a5a131b983552e085e6ba89a07fe4f5b
+size 14244
diff --git a/checkpoint-100/scaler.pt b/checkpoint-100/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6b3b7531b67d6a3286c705af0b9b33aed1a3dd8
--- /dev/null
+++ b/checkpoint-100/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4a558c5b93581a5c41e2922404490319bcd15a10296c927a72a41012ff7f27
+size 988
diff --git a/checkpoint-100/scheduler.pt b/checkpoint-100/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba43507aebfbeb306a5d19f9fb9d5fc27a23cf0b
--- /dev/null
+++ b/checkpoint-100/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c869bfb8f14e59279afcb8a8eecd370b83ad9128cc999745f22a1b121cdf645
+size 1064
diff --git a/checkpoint-100/tokenizer.json b/checkpoint-100/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-100/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-100/tokenizer_config.json b/checkpoint-100/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-100/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-100/trainer_state.json b/checkpoint-100/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9732e82e7fcc4032ecac72aa0a54f4c05bd6f411
--- /dev/null
+++ b/checkpoint-100/trainer_state.json
@@ -0,0 +1,2834 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.8130081300813008,
+ "eval_steps": 500,
+ "global_step": 100,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 411488,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-100/training_args.bin b/checkpoint-100/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-100/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-150/README.md b/checkpoint-150/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-150/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-150/adapter_config.json b/checkpoint-150/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-150/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-150/adapter_model.safetensors b/checkpoint-150/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a1502e3904dec405908eea76528099da99eb542a
--- /dev/null
+++ b/checkpoint-150/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c4e9a9589a7474f445663b6ff20677982b29ec3c7c1e0aaaf1ee48fbfad4ad5
+size 8731128
diff --git a/checkpoint-150/chat_template.jinja b/checkpoint-150/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-150/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-150/optimizer.pt b/checkpoint-150/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a16aa2d6d287e87c5ec87e9d597e417180bd9b1
--- /dev/null
+++ b/checkpoint-150/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6beaea033a70db5b5a9402e7fcb22cd071216c551154452086b4d9eccaafd53
+size 17526842
diff --git a/checkpoint-150/ref/adapter_config.json b/checkpoint-150/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-150/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-150/ref/adapter_model.safetensors b/checkpoint-150/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-150/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-150/rng_state.pth b/checkpoint-150/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..759cb39a66adc9b479d54948cfa177fe6218bf5d
--- /dev/null
+++ b/checkpoint-150/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27656c488218789eb6a077238f1346863125281337293d34eaf84a1aba2638e8
+size 14244
diff --git a/checkpoint-150/scaler.pt b/checkpoint-150/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51187ca97e9b84514892b89e13cb0f9671857520
--- /dev/null
+++ b/checkpoint-150/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da800fd5ac660f36778294f9df633c4f039b5f7bde42ca6a92dea5ac7d9666b1
+size 988
diff --git a/checkpoint-150/scheduler.pt b/checkpoint-150/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9e395f2702dc71dbebfa2593df822fb1fcd96a9
--- /dev/null
+++ b/checkpoint-150/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead0c69f231af9ff216a15725240624f552cc5e549122accf41fc9de1475d582
+size 1064
diff --git a/checkpoint-150/tokenizer.json b/checkpoint-150/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-150/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-150/tokenizer_config.json b/checkpoint-150/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-150/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-150/trainer_state.json b/checkpoint-150/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..259c5cc6f9dea9547cbf25e4816fbe22c568375d
--- /dev/null
+++ b/checkpoint-150/trainer_state.json
@@ -0,0 +1,4234 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.2195121951219512,
+ "eval_steps": 500,
+ "global_step": 150,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 619439,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-150/training_args.bin b/checkpoint-150/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-150/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-200/README.md b/checkpoint-200/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-200/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-200/adapter_config.json b/checkpoint-200/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-200/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-200/adapter_model.safetensors b/checkpoint-200/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..08a2c3e7ec2105e12a84d9cd57207946ec7a7573
--- /dev/null
+++ b/checkpoint-200/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3760d3314549081af05923853132db52f91e157452d1a671fa01138e8ce0d7f6
+size 8731128
diff --git a/checkpoint-200/chat_template.jinja b/checkpoint-200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-200/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-200/optimizer.pt b/checkpoint-200/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3a0d4e01f9555d126290e96a3d6470b7557b4c7
--- /dev/null
+++ b/checkpoint-200/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a433c0070b886ab356b3b0e01ac77896089a29f3e1654b8040ce587b35da19b
+size 17526842
diff --git a/checkpoint-200/ref/adapter_config.json b/checkpoint-200/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-200/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-200/ref/adapter_model.safetensors b/checkpoint-200/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-200/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-200/rng_state.pth b/checkpoint-200/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..23ee49c41a13b857dd690cbd634f7d0f10e60508
--- /dev/null
+++ b/checkpoint-200/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4825bf31044ed1c1141fad5e2d04ce4a1b0a47c2a63289a50920eb60f313e05
+size 14244
diff --git a/checkpoint-200/scaler.pt b/checkpoint-200/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62cc01b88d92c6635037c93f26205863eb8854bc
--- /dev/null
+++ b/checkpoint-200/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bd55b3848d82967a207e0805911c79200c6adce71e3b37fd24549a718f75738
+size 988
diff --git a/checkpoint-200/scheduler.pt b/checkpoint-200/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a081c242fdc3d41cb596d73cee2ca70aa1e1efe8
--- /dev/null
+++ b/checkpoint-200/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820dc20ea8497c148d31704bc1cfea7c0d598f4fe3e52b035416778336d712a4
+size 1064
diff --git a/checkpoint-200/tokenizer.json b/checkpoint-200/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-200/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-200/tokenizer_config.json b/checkpoint-200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-200/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-200/trainer_state.json b/checkpoint-200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e50dab06b25d3c046c838d74e61d2f515cd2401
--- /dev/null
+++ b/checkpoint-200/trainer_state.json
@@ -0,0 +1,5634 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.6260162601626016,
+ "eval_steps": 500,
+ "global_step": 200,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1402934789657593,
+ "epoch": 1.2276422764227641,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010795716661959887,
+ "kl": 1.4652535810455447e-05,
+ "learning_rate": 5.923033215939834e-07,
+ "loss": 7.542968205598299e-07,
+ "num_tokens": 621009.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 151,
+ "step_time": 3.926544339999964
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.190350890159607,
+ "epoch": 1.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19853363931179047,
+ "kl": 2.269768037876929e-05,
+ "learning_rate": 5.889758273918683e-07,
+ "loss": 0.044217392802238464,
+ "num_tokens": 623994.0,
+ "reward": 0.4411996603012085,
+ "reward_std": 0.2517909109592438,
+ "rewards/true_env_reward_fn/mean": 0.4411996603012085,
+ "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "step": 152,
+ "step_time": 3.7339736520007136
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2316884994506836,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.35867181699723e-05,
+ "kl": 1.1579370038816705e-05,
+ "learning_rate": 5.856314121676467e-07,
+ "loss": 5.79387460675207e-07,
+ "num_tokens": 628224.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 153,
+ "step_time": 4.375236807001784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4028943181037903,
+ "epoch": 1.2520325203252032,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25418931245803833,
+ "kl": 3.6890452065563295e-05,
+ "learning_rate": 5.822703753824966e-07,
+ "loss": 0.25599968433380127,
+ "num_tokens": 631183.0,
+ "reward": 0.38683533668518066,
+ "reward_std": 0.43613559007644653,
+ "rewards/true_env_reward_fn/mean": 0.38683533668518066,
+ "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "step": 154,
+ "step_time": 6.055355972999678
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.367663562297821,
+ "epoch": 1.2601626016260163,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331811130046844,
+ "kl": 1.2863993106293492e-05,
+ "learning_rate": 5.788930179859024e-07,
+ "loss": -0.0047044456005096436,
+ "num_tokens": 636230.0,
+ "reward": 0.3489508628845215,
+ "reward_std": 0.13627417385578156,
+ "rewards/true_env_reward_fn/mean": 0.3489508628845215,
+ "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "step": 155,
+ "step_time": 3.9174396130001696
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 68.5,
+ "completions/mean_terminated_length": 68.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2759611010551453,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12738144397735596,
+ "kl": 1.5844128029129934e-05,
+ "learning_rate": 5.754996423887061e-07,
+ "loss": -0.011055335402488708,
+ "num_tokens": 640262.0,
+ "reward": 0.344404935836792,
+ "reward_std": 0.16542991995811462,
+ "rewards/true_env_reward_fn/mean": 0.344404935836792,
+ "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "step": 156,
+ "step_time": 5.706334413998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0904476642608643,
+ "epoch": 1.2764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.733699021628127e-05,
+ "kl": 1.2238857834745431e-05,
+ "learning_rate": 5.720905524360308e-07,
+ "loss": 6.076299996493617e-07,
+ "num_tokens": 645091.0,
+ "reward": 0.4731999933719635,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4731999933719635,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 157,
+ "step_time": 3.7696847109982627
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.0,
+ "completions/mean_terminated_length": 60.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3856677412986755,
+ "epoch": 1.2845528455284554,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18670028448104858,
+ "kl": 1.8415606064081658e-05,
+ "learning_rate": 5.686660533800736e-07,
+ "loss": -0.07078710198402405,
+ "num_tokens": 648179.0,
+ "reward": 0.537517786026001,
+ "reward_std": 0.1451217085123062,
+ "rewards/true_env_reward_fn/mean": 0.537517786026001,
+ "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "step": 158,
+ "step_time": 3.7075291149994882
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 68.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.121916651725769,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11415883898735046,
+ "kl": 1.909901220642496e-05,
+ "learning_rate": 5.652264518527725e-07,
+ "loss": -0.04401372745633125,
+ "num_tokens": 652044.0,
+ "reward": 0.5182899832725525,
+ "reward_std": 0.21869486570358276,
+ "rewards/true_env_reward_fn/mean": 0.5182899832725525,
+ "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "step": 159,
+ "step_time": 3.8929355969994504
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5205118060112,
+ "epoch": 1.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2403375506401062,
+ "kl": 3.7574073758150917e-05,
+ "learning_rate": 5.617720558383508e-07,
+ "loss": 0.26385918259620667,
+ "num_tokens": 656362.0,
+ "reward": 0.41201668977737427,
+ "reward_std": 0.2023741453886032,
+ "rewards/true_env_reward_fn/mean": 0.41201668977737427,
+ "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "step": 160,
+ "step_time": 6.157555950998358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3556928038597107,
+ "epoch": 1.3089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18135924637317657,
+ "kl": 2.6372636057203636e-05,
+ "learning_rate": 5.583031746457407e-07,
+ "loss": -0.10538280755281448,
+ "num_tokens": 659977.0,
+ "reward": 0.4239906072616577,
+ "reward_std": 0.3287450969219208,
+ "rewards/true_env_reward_fn/mean": 0.4239906072616577,
+ "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "step": 161,
+ "step_time": 4.709477423999488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 52.5,
+ "completions/mean_terminated_length": 52.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 0.9743769466876984,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.178797647356987,
+ "kl": 1.2532927030406427e-05,
+ "learning_rate": 5.548201188808869e-07,
+ "loss": -0.04164513945579529,
+ "num_tokens": 661409.0,
+ "reward": 0.8041956424713135,
+ "reward_std": 0.1363772451877594,
+ "rewards/true_env_reward_fn/mean": 0.8041956424713135,
+ "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "step": 162,
+ "step_time": 3.1791253910014348
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.3072250485420227,
+ "epoch": 1.3252032520325203,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16768279671669006,
+ "kl": 1.89386219062726e-05,
+ "learning_rate": 5.513232004189339e-07,
+ "loss": -0.01292814314365387,
+ "num_tokens": 666504.0,
+ "reward": 0.27981066703796387,
+ "reward_std": 0.2949208915233612,
+ "rewards/true_env_reward_fn/mean": 0.27981066703796387,
+ "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "step": 163,
+ "step_time": 3.778431355000066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 51.0,
+ "completions/mean_terminated_length": 51.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1984856128692627,
+ "epoch": 1.3333333333333333,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13123764097690582,
+ "kl": 1.9091786271019373e-05,
+ "learning_rate": 5.478127323763027e-07,
+ "loss": 0.035523779690265656,
+ "num_tokens": 671004.0,
+ "reward": 0.2771500051021576,
+ "reward_std": 0.30146247148513794,
+ "rewards/true_env_reward_fn/mean": 0.2771500051021576,
+ "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "step": 164,
+ "step_time": 3.304021460000513
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 84.875,
+ "completions/mean_terminated_length": 84.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2726752758026123,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15594874322414398,
+ "kl": 2.5981638827943243e-05,
+ "learning_rate": 5.442890290826518e-07,
+ "loss": -0.01398652046918869,
+ "num_tokens": 677307.0,
+ "reward": 0.35573017597198486,
+ "reward_std": 0.25944042205810547,
+ "rewards/true_env_reward_fn/mean": 0.35573017597198486,
+ "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "step": 165,
+ "step_time": 4.972808451999299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 103.625,
+ "completions/mean_terminated_length": 103.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0949090719223022,
+ "epoch": 1.3495934959349594,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07191785424947739,
+ "kl": 1.4828182884230046e-05,
+ "learning_rate": 5.407524060527332e-07,
+ "loss": -0.05063021928071976,
+ "num_tokens": 682076.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 166,
+ "step_time": 8.769379133000257
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1857684254646301,
+ "epoch": 1.3577235772357723,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.923542191041633e-05,
+ "kl": 1.0165251751459436e-05,
+ "learning_rate": 5.37203179958141e-07,
+ "loss": 5.114516170579009e-07,
+ "num_tokens": 685500.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 167,
+ "step_time": 3.1906087530005607
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 74.25,
+ "completions/mean_terminated_length": 74.25,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.527149498462677,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13164250552654266,
+ "kl": 1.8541333702160046e-05,
+ "learning_rate": 5.33641668598956e-07,
+ "loss": -0.2347300797700882,
+ "num_tokens": 688318.0,
+ "reward": 0.7218117713928223,
+ "reward_std": 0.1818692982196808,
+ "rewards/true_env_reward_fn/mean": 0.7218117713928223,
+ "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "step": 168,
+ "step_time": 8.50137474999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 63.0,
+ "completions/min_terminated_length": 63.0,
+ "entropy": 1.3749513030052185,
+ "epoch": 1.3739837398373984,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13246971368789673,
+ "kl": 1.2620409506780561e-05,
+ "learning_rate": 5.300681908752895e-07,
+ "loss": 0.024534843862056732,
+ "num_tokens": 692541.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 169,
+ "step_time": 3.9512340759993094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2032299041748047,
+ "epoch": 1.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11185794323682785,
+ "kl": 1.5517784049734473e-05,
+ "learning_rate": 5.264830667587295e-07,
+ "loss": -0.05245225131511688,
+ "num_tokens": 698064.0,
+ "reward": 0.4444866180419922,
+ "reward_std": 0.32400256395339966,
+ "rewards/true_env_reward_fn/mean": 0.4444866180419922,
+ "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "step": 170,
+ "step_time": 4.471538110999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 95.0,
+ "completions/mean_terminated_length": 95.0,
+ "completions/min_length": 79.0,
+ "completions/min_terminated_length": 79.0,
+ "entropy": 1.4204387068748474,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1646534949541092,
+ "kl": 2.4697198568901513e-05,
+ "learning_rate": 5.228866172636899e-07,
+ "loss": 0.02632315456867218,
+ "num_tokens": 704196.0,
+ "reward": 0.304565966129303,
+ "reward_std": 0.32997164130210876,
+ "rewards/true_env_reward_fn/mean": 0.304565966129303,
+ "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "step": 171,
+ "step_time": 5.0436168590003945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1254178285598755,
+ "epoch": 1.3983739837398375,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15058821439743042,
+ "kl": 1.8407325114822015e-05,
+ "learning_rate": 5.192791644186662e-07,
+ "loss": 0.025478817522525787,
+ "num_tokens": 706411.0,
+ "reward": 0.7279239892959595,
+ "reward_std": 0.11376125365495682,
+ "rewards/true_env_reward_fn/mean": 0.7279239892959595,
+ "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "step": 172,
+ "step_time": 3.2261944119982218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0894773602485657,
+ "epoch": 1.4065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1687694787979126,
+ "kl": 2.6046765015053097e-05,
+ "learning_rate": 5.156610312374013e-07,
+ "loss": -0.056941211223602295,
+ "num_tokens": 711212.0,
+ "reward": 0.4907146692276001,
+ "reward_std": 0.3376546800136566,
+ "rewards/true_env_reward_fn/mean": 0.4907146692276001,
+ "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "step": 173,
+ "step_time": 3.9498180619993946
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 60.125,
+ "completions/mean_terminated_length": 60.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2238691449165344,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19584441184997559,
+ "kl": 4.8285241064149886e-05,
+ "learning_rate": 5.120325416899629e-07,
+ "loss": 0.0766875222325325,
+ "num_tokens": 715409.0,
+ "reward": 0.4593355059623718,
+ "reward_std": 0.3909546136856079,
+ "rewards/true_env_reward_fn/mean": 0.4593355059623718,
+ "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "step": 174,
+ "step_time": 4.100519798999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1763202548027039,
+ "epoch": 1.4227642276422765,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1448555737733841,
+ "kl": 1.2618989785551094e-05,
+ "learning_rate": 5.08394020673734e-07,
+ "loss": -0.012558378279209137,
+ "num_tokens": 722327.0,
+ "reward": 0.15966665744781494,
+ "reward_std": 0.3235519230365753,
+ "rewards/true_env_reward_fn/mean": 0.15966665744781494,
+ "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "step": 175,
+ "step_time": 3.875348296000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2852763533592224,
+ "epoch": 1.4308943089430894,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22303813695907593,
+ "kl": 4.580334098136518e-05,
+ "learning_rate": 5.047457939843227e-07,
+ "loss": -0.09214464575052261,
+ "num_tokens": 726828.0,
+ "reward": 0.25830498337745667,
+ "reward_std": 0.37860655784606934,
+ "rewards/true_env_reward_fn/mean": 0.25830498337745667,
+ "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "step": 176,
+ "step_time": 4.241473076999682
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3609731197357178,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.001985745271668e-05,
+ "kl": 1.4942165307729738e-05,
+ "learning_rate": 5.010881882863893e-07,
+ "loss": 7.44550789022469e-07,
+ "num_tokens": 729930.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 177,
+ "step_time": 3.3902666960002534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2678966522216797,
+ "epoch": 1.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15351757407188416,
+ "kl": 1.6737100395403104e-05,
+ "learning_rate": 4.974215310843967e-07,
+ "loss": 0.041131969541311264,
+ "num_tokens": 732155.0,
+ "reward": 0.7803820371627808,
+ "reward_std": 0.08667682856321335,
+ "rewards/true_env_reward_fn/mean": 0.7803820371627808,
+ "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "step": 178,
+ "step_time": 3.637111981999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.146271526813507,
+ "epoch": 1.4552845528455285,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18466196954250336,
+ "kl": 3.719841197380447e-05,
+ "learning_rate": 4.937461506932859e-07,
+ "loss": 0.029051154851913452,
+ "num_tokens": 735418.0,
+ "reward": 0.40377071499824524,
+ "reward_std": 0.28345924615859985,
+ "rewards/true_env_reward_fn/mean": 0.40377071499824524,
+ "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "step": 179,
+ "step_time": 3.340555791999577
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.3902945518493652,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1020251139998436,
+ "kl": 1.8220500351162627e-05,
+ "learning_rate": 4.900623762090777e-07,
+ "loss": -0.002344265580177307,
+ "num_tokens": 740540.0,
+ "reward": 0.3590222895145416,
+ "reward_std": 0.12487777322530746,
+ "rewards/true_env_reward_fn/mean": 0.3590222895145416,
+ "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "step": 180,
+ "step_time": 7.219923718001155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.517557680606842,
+ "epoch": 1.4715447154471546,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24192294478416443,
+ "kl": 4.1268089262302965e-05,
+ "learning_rate": 4.863705374794055e-07,
+ "loss": 0.09132950007915497,
+ "num_tokens": 744723.0,
+ "reward": 0.23991000652313232,
+ "reward_std": 0.2837013602256775,
+ "rewards/true_env_reward_fn/mean": 0.23991000652313232,
+ "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "step": 181,
+ "step_time": 3.696339096999509
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.249614655971527,
+ "epoch": 1.4796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1566745787858963,
+ "kl": 2.6629099920683075e-05,
+ "learning_rate": 4.826709650739811e-07,
+ "loss": 0.003972277045249939,
+ "num_tokens": 748979.0,
+ "reward": 0.4935140311717987,
+ "reward_std": 0.41978561878204346,
+ "rewards/true_env_reward_fn/mean": 0.4935140311717987,
+ "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "step": 182,
+ "step_time": 3.316512920000605
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1666916608810425,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001005820304271765,
+ "kl": 1.2246940059412736e-05,
+ "learning_rate": 4.789639902549948e-07,
+ "loss": 6.330609494398232e-07,
+ "num_tokens": 751323.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 183,
+ "step_time": 3.7099916660008603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3246222138404846,
+ "epoch": 1.4959349593495934,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18140996992588043,
+ "kl": 3.1042441150930244e-05,
+ "learning_rate": 4.752499449474535e-07,
+ "loss": -0.022353097796440125,
+ "num_tokens": 755494.0,
+ "reward": 0.4561777412891388,
+ "reward_std": 0.2439236342906952,
+ "rewards/true_env_reward_fn/mean": 0.4561777412891388,
+ "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "step": 184,
+ "step_time": 3.7916486710000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 75.75,
+ "completions/mean_terminated_length": 75.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.2625707983970642,
+ "epoch": 1.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11702944338321686,
+ "kl": 1.503958355897339e-05,
+ "learning_rate": 4.715291617094607e-07,
+ "loss": 0.023916304111480713,
+ "num_tokens": 758432.0,
+ "reward": 0.5015827417373657,
+ "reward_std": 0.1783808320760727,
+ "rewards/true_env_reward_fn/mean": 0.5015827417373657,
+ "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "step": 185,
+ "step_time": 4.226409274000616
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1460023522377014,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11851620674133301,
+ "kl": 1.3728345948038623e-05,
+ "learning_rate": 4.678019737024387e-07,
+ "loss": 0.0831337422132492,
+ "num_tokens": 764336.0,
+ "reward": 0.19094166159629822,
+ "reward_std": 0.30934420228004456,
+ "rewards/true_env_reward_fn/mean": 0.19094166159629822,
+ "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "step": 186,
+ "step_time": 3.563357556000483
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1291148662567139,
+ "epoch": 1.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.864256960805506e-05,
+ "kl": 1.081683785741916e-05,
+ "learning_rate": 4.6406871466129704e-07,
+ "loss": 5.412177870312007e-07,
+ "num_tokens": 766608.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 187,
+ "step_time": 3.534869859002356
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 92.0,
+ "completions/mean_terminated_length": 92.0,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3645328283309937,
+ "epoch": 1.5284552845528454,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18168245255947113,
+ "kl": 2.6857565899263136e-05,
+ "learning_rate": 4.6032971886454956e-07,
+ "loss": -0.005156125873327255,
+ "num_tokens": 774496.0,
+ "reward": 0.11249999701976776,
+ "reward_std": 0.20856082439422607,
+ "rewards/true_env_reward_fn/mean": 0.11249999701976776,
+ "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "step": 188,
+ "step_time": 8.34005261099992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1495982110500336,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14486820995807648,
+ "kl": 1.3202762147557223e-05,
+ "learning_rate": 4.5658532110438337e-07,
+ "loss": -0.0010610297322273254,
+ "num_tokens": 777186.0,
+ "reward": 0.5879127383232117,
+ "reward_std": 0.05142820253968239,
+ "rewards/true_env_reward_fn/mean": 0.5879127383232117,
+ "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "step": 189,
+ "step_time": 3.4856022139993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.241140365600586,
+ "epoch": 1.5447154471544715,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001250067143701017,
+ "kl": 1.5482702110602986e-05,
+ "learning_rate": 4.52835856656681e-07,
+ "loss": 7.80837922320643e-07,
+ "num_tokens": 779965.0,
+ "reward": 0.6861198544502258,
+ "reward_std": 0.08807206153869629,
+ "rewards/true_env_reward_fn/mean": 0.6861198544502258,
+ "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "step": 190,
+ "step_time": 3.904181735999373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2944807410240173,
+ "epoch": 1.5528455284552845,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2030111849308014,
+ "kl": 3.585523518268019e-05,
+ "learning_rate": 4.490816612509991e-07,
+ "loss": 0.0143373291939497,
+ "num_tokens": 786140.0,
+ "reward": 0.35173332691192627,
+ "reward_std": 0.18115806579589844,
+ "rewards/true_env_reward_fn/mean": 0.35173332691192627,
+ "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "step": 191,
+ "step_time": 4.005758510000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 58.0,
+ "completions/mean_terminated_length": 58.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2177271246910095,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11188288033008575,
+ "kl": 2.3622495064046234e-05,
+ "learning_rate": 4.45323071040508e-07,
+ "loss": -0.02369789034128189,
+ "num_tokens": 790424.0,
+ "reward": 0.4888629913330078,
+ "reward_std": 0.23310808837413788,
+ "rewards/true_env_reward_fn/mean": 0.4888629913330078,
+ "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "step": 192,
+ "step_time": 3.2518814809991454
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.192966103553772,
+ "epoch": 1.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1378823071718216,
+ "kl": 1.7358055174554465e-05,
+ "learning_rate": 4.4156042257189143e-07,
+ "loss": 0.06256310641765594,
+ "num_tokens": 794521.0,
+ "reward": 0.5219699740409851,
+ "reward_std": 0.06214587390422821,
+ "rewards/true_env_reward_fn/mean": 0.5219699740409851,
+ "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "step": 193,
+ "step_time": 4.222739491999164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1770159006118774,
+ "epoch": 1.5772357723577235,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19550059735774994,
+ "kl": 2.4871268578863237e-05,
+ "learning_rate": 4.377940527552125e-07,
+ "loss": 0.05841376632452011,
+ "num_tokens": 798194.0,
+ "reward": 0.42302167415618896,
+ "reward_std": 0.2911272346973419,
+ "rewards/true_env_reward_fn/mean": 0.42302167415618896,
+ "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "step": 194,
+ "step_time": 3.8170270639984665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.5244255661964417,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1872202306985855,
+ "kl": 2.5990483663917985e-05,
+ "learning_rate": 4.340242988337462e-07,
+ "loss": -0.044112429022789,
+ "num_tokens": 802802.0,
+ "reward": 0.33745431900024414,
+ "reward_std": 0.22955451905727386,
+ "rewards/true_env_reward_fn/mean": 0.33745431900024414,
+ "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "step": 195,
+ "step_time": 4.27381555500142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.345891296863556,
+ "epoch": 1.5934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16891054809093475,
+ "kl": 2.7261638024356216e-05,
+ "learning_rate": 4.3025149835378275e-07,
+ "loss": -0.139386385679245,
+ "num_tokens": 807881.0,
+ "reward": 0.3240283131599426,
+ "reward_std": 0.2803676128387451,
+ "rewards/true_env_reward_fn/mean": 0.3240283131599426,
+ "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "step": 196,
+ "step_time": 4.981287381999209
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.246802031993866,
+ "epoch": 1.6016260162601625,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2096729576587677,
+ "kl": 3.5958016269432846e-05,
+ "learning_rate": 4.2647598913440264e-07,
+ "loss": -0.02941281348466873,
+ "num_tokens": 812500.0,
+ "reward": 0.4126526415348053,
+ "reward_std": 0.36393746733665466,
+ "rewards/true_env_reward_fn/mean": 0.4126526415348053,
+ "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "step": 197,
+ "step_time": 3.101726017999681
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4369062185287476,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1862105429172516,
+ "kl": 4.7646244638599455e-05,
+ "learning_rate": 4.2269810923722965e-07,
+ "loss": 0.0521145761013031,
+ "num_tokens": 818718.0,
+ "reward": 0.26869943737983704,
+ "reward_std": 0.1483483463525772,
+ "rewards/true_env_reward_fn/mean": 0.26869943737983704,
+ "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "step": 198,
+ "step_time": 4.287780451000799
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.192937195301056,
+ "epoch": 1.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2013663798570633,
+ "kl": 1.8760739294521045e-05,
+ "learning_rate": 4.189181969361588e-07,
+ "loss": 0.07236722111701965,
+ "num_tokens": 825728.0,
+ "reward": 0.23110000789165497,
+ "reward_std": 0.23212090134620667,
+ "rewards/true_env_reward_fn/mean": 0.23110000789165497,
+ "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "step": 199,
+ "step_time": 4.708717262998107
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.3792839050292969,
+ "epoch": 1.6260162601626016,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.7985117463395e-05,
+ "kl": 1.4280476534622721e-05,
+ "learning_rate": 4.1513659068706814e-07,
+ "loss": 7.153485626076872e-07,
+ "num_tokens": 830318.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 200,
+ "step_time": 3.9055351140013954
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 830318,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-200/training_args.bin b/checkpoint-200/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-200/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-250/README.md b/checkpoint-250/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-250/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-250/adapter_config.json b/checkpoint-250/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-250/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-250/adapter_model.safetensors b/checkpoint-250/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..436df24b34707027ac6d3abf5ca3b3f131bbd728
--- /dev/null
+++ b/checkpoint-250/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:778e2eb5058363271c6371ce49aa6d59c2c58ff759e8dc2c50f9f2c8378caa86
+size 8731128
diff --git a/checkpoint-250/chat_template.jinja b/checkpoint-250/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-250/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-250/optimizer.pt b/checkpoint-250/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ec03e7f1a862d34a2b4f789cdbfd8d6d406b92e
--- /dev/null
+++ b/checkpoint-250/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f6c9d8b5c73cdba61371997fc1aa84a8aa321b3b3587dfbdaa6c33b016269d3
+size 17526842
diff --git a/checkpoint-250/ref/adapter_config.json b/checkpoint-250/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-250/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-250/ref/adapter_model.safetensors b/checkpoint-250/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-250/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-250/rng_state.pth b/checkpoint-250/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b165d5b908fe58578b31b49bfb858eeb56018301
--- /dev/null
+++ b/checkpoint-250/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6117437beb154fdbe42da769bc751661179f0830057abcaeb1073a915d11a232
+size 14244
diff --git a/checkpoint-250/scaler.pt b/checkpoint-250/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e81262ee4c5b310cd1dff52da63a5aa8c200b8cb
--- /dev/null
+++ b/checkpoint-250/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccec82ecb1db9fee8431a9924525ac42c65eba373a478c5528e653b68744a6a9
+size 988
diff --git a/checkpoint-250/scheduler.pt b/checkpoint-250/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3be729a321146873c53daa0d0e4d870636afc87
--- /dev/null
+++ b/checkpoint-250/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40db5c7e3b57ff5ba822a7ba8434e7a868d756b70556c6f711c44f3bda3fde48
+size 1064
diff --git a/checkpoint-250/tokenizer.json b/checkpoint-250/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-250/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-250/tokenizer_config.json b/checkpoint-250/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-250/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-250/trainer_state.json b/checkpoint-250/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c91fac69447982f13bbebe56ba80c2e8f86ade45
--- /dev/null
+++ b/checkpoint-250/trainer_state.json
@@ -0,0 +1,7034 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.032520325203252,
+ "eval_steps": 500,
+ "global_step": 250,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1402934789657593,
+ "epoch": 1.2276422764227641,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010795716661959887,
+ "kl": 1.4652535810455447e-05,
+ "learning_rate": 5.923033215939834e-07,
+ "loss": 7.542968205598299e-07,
+ "num_tokens": 621009.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 151,
+ "step_time": 3.926544339999964
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.190350890159607,
+ "epoch": 1.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19853363931179047,
+ "kl": 2.269768037876929e-05,
+ "learning_rate": 5.889758273918683e-07,
+ "loss": 0.044217392802238464,
+ "num_tokens": 623994.0,
+ "reward": 0.4411996603012085,
+ "reward_std": 0.2517909109592438,
+ "rewards/true_env_reward_fn/mean": 0.4411996603012085,
+ "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "step": 152,
+ "step_time": 3.7339736520007136
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2316884994506836,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.35867181699723e-05,
+ "kl": 1.1579370038816705e-05,
+ "learning_rate": 5.856314121676467e-07,
+ "loss": 5.79387460675207e-07,
+ "num_tokens": 628224.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 153,
+ "step_time": 4.375236807001784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4028943181037903,
+ "epoch": 1.2520325203252032,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25418931245803833,
+ "kl": 3.6890452065563295e-05,
+ "learning_rate": 5.822703753824966e-07,
+ "loss": 0.25599968433380127,
+ "num_tokens": 631183.0,
+ "reward": 0.38683533668518066,
+ "reward_std": 0.43613559007644653,
+ "rewards/true_env_reward_fn/mean": 0.38683533668518066,
+ "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "step": 154,
+ "step_time": 6.055355972999678
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.367663562297821,
+ "epoch": 1.2601626016260163,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331811130046844,
+ "kl": 1.2863993106293492e-05,
+ "learning_rate": 5.788930179859024e-07,
+ "loss": -0.0047044456005096436,
+ "num_tokens": 636230.0,
+ "reward": 0.3489508628845215,
+ "reward_std": 0.13627417385578156,
+ "rewards/true_env_reward_fn/mean": 0.3489508628845215,
+ "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "step": 155,
+ "step_time": 3.9174396130001696
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 68.5,
+ "completions/mean_terminated_length": 68.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2759611010551453,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12738144397735596,
+ "kl": 1.5844128029129934e-05,
+ "learning_rate": 5.754996423887061e-07,
+ "loss": -0.011055335402488708,
+ "num_tokens": 640262.0,
+ "reward": 0.344404935836792,
+ "reward_std": 0.16542991995811462,
+ "rewards/true_env_reward_fn/mean": 0.344404935836792,
+ "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "step": 156,
+ "step_time": 5.706334413998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0904476642608643,
+ "epoch": 1.2764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.733699021628127e-05,
+ "kl": 1.2238857834745431e-05,
+ "learning_rate": 5.720905524360308e-07,
+ "loss": 6.076299996493617e-07,
+ "num_tokens": 645091.0,
+ "reward": 0.4731999933719635,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4731999933719635,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 157,
+ "step_time": 3.7696847109982627
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.0,
+ "completions/mean_terminated_length": 60.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3856677412986755,
+ "epoch": 1.2845528455284554,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18670028448104858,
+ "kl": 1.8415606064081658e-05,
+ "learning_rate": 5.686660533800736e-07,
+ "loss": -0.07078710198402405,
+ "num_tokens": 648179.0,
+ "reward": 0.537517786026001,
+ "reward_std": 0.1451217085123062,
+ "rewards/true_env_reward_fn/mean": 0.537517786026001,
+ "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "step": 158,
+ "step_time": 3.7075291149994882
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 68.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.121916651725769,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11415883898735046,
+ "kl": 1.909901220642496e-05,
+ "learning_rate": 5.652264518527725e-07,
+ "loss": -0.04401372745633125,
+ "num_tokens": 652044.0,
+ "reward": 0.5182899832725525,
+ "reward_std": 0.21869486570358276,
+ "rewards/true_env_reward_fn/mean": 0.5182899832725525,
+ "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "step": 159,
+ "step_time": 3.8929355969994504
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5205118060112,
+ "epoch": 1.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2403375506401062,
+ "kl": 3.7574073758150917e-05,
+ "learning_rate": 5.617720558383508e-07,
+ "loss": 0.26385918259620667,
+ "num_tokens": 656362.0,
+ "reward": 0.41201668977737427,
+ "reward_std": 0.2023741453886032,
+ "rewards/true_env_reward_fn/mean": 0.41201668977737427,
+ "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "step": 160,
+ "step_time": 6.157555950998358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3556928038597107,
+ "epoch": 1.3089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18135924637317657,
+ "kl": 2.6372636057203636e-05,
+ "learning_rate": 5.583031746457407e-07,
+ "loss": -0.10538280755281448,
+ "num_tokens": 659977.0,
+ "reward": 0.4239906072616577,
+ "reward_std": 0.3287450969219208,
+ "rewards/true_env_reward_fn/mean": 0.4239906072616577,
+ "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "step": 161,
+ "step_time": 4.709477423999488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 52.5,
+ "completions/mean_terminated_length": 52.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 0.9743769466876984,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.178797647356987,
+ "kl": 1.2532927030406427e-05,
+ "learning_rate": 5.548201188808869e-07,
+ "loss": -0.04164513945579529,
+ "num_tokens": 661409.0,
+ "reward": 0.8041956424713135,
+ "reward_std": 0.1363772451877594,
+ "rewards/true_env_reward_fn/mean": 0.8041956424713135,
+ "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "step": 162,
+ "step_time": 3.1791253910014348
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.3072250485420227,
+ "epoch": 1.3252032520325203,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16768279671669006,
+ "kl": 1.89386219062726e-05,
+ "learning_rate": 5.513232004189339e-07,
+ "loss": -0.01292814314365387,
+ "num_tokens": 666504.0,
+ "reward": 0.27981066703796387,
+ "reward_std": 0.2949208915233612,
+ "rewards/true_env_reward_fn/mean": 0.27981066703796387,
+ "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "step": 163,
+ "step_time": 3.778431355000066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 51.0,
+ "completions/mean_terminated_length": 51.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1984856128692627,
+ "epoch": 1.3333333333333333,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13123764097690582,
+ "kl": 1.9091786271019373e-05,
+ "learning_rate": 5.478127323763027e-07,
+ "loss": 0.035523779690265656,
+ "num_tokens": 671004.0,
+ "reward": 0.2771500051021576,
+ "reward_std": 0.30146247148513794,
+ "rewards/true_env_reward_fn/mean": 0.2771500051021576,
+ "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "step": 164,
+ "step_time": 3.304021460000513
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 84.875,
+ "completions/mean_terminated_length": 84.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2726752758026123,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15594874322414398,
+ "kl": 2.5981638827943243e-05,
+ "learning_rate": 5.442890290826518e-07,
+ "loss": -0.01398652046918869,
+ "num_tokens": 677307.0,
+ "reward": 0.35573017597198486,
+ "reward_std": 0.25944042205810547,
+ "rewards/true_env_reward_fn/mean": 0.35573017597198486,
+ "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "step": 165,
+ "step_time": 4.972808451999299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 103.625,
+ "completions/mean_terminated_length": 103.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0949090719223022,
+ "epoch": 1.3495934959349594,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07191785424947739,
+ "kl": 1.4828182884230046e-05,
+ "learning_rate": 5.407524060527332e-07,
+ "loss": -0.05063021928071976,
+ "num_tokens": 682076.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 166,
+ "step_time": 8.769379133000257
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1857684254646301,
+ "epoch": 1.3577235772357723,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.923542191041633e-05,
+ "kl": 1.0165251751459436e-05,
+ "learning_rate": 5.37203179958141e-07,
+ "loss": 5.114516170579009e-07,
+ "num_tokens": 685500.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 167,
+ "step_time": 3.1906087530005607
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 74.25,
+ "completions/mean_terminated_length": 74.25,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.527149498462677,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13164250552654266,
+ "kl": 1.8541333702160046e-05,
+ "learning_rate": 5.33641668598956e-07,
+ "loss": -0.2347300797700882,
+ "num_tokens": 688318.0,
+ "reward": 0.7218117713928223,
+ "reward_std": 0.1818692982196808,
+ "rewards/true_env_reward_fn/mean": 0.7218117713928223,
+ "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "step": 168,
+ "step_time": 8.50137474999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 63.0,
+ "completions/min_terminated_length": 63.0,
+ "entropy": 1.3749513030052185,
+ "epoch": 1.3739837398373984,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13246971368789673,
+ "kl": 1.2620409506780561e-05,
+ "learning_rate": 5.300681908752895e-07,
+ "loss": 0.024534843862056732,
+ "num_tokens": 692541.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 169,
+ "step_time": 3.9512340759993094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2032299041748047,
+ "epoch": 1.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11185794323682785,
+ "kl": 1.5517784049734473e-05,
+ "learning_rate": 5.264830667587295e-07,
+ "loss": -0.05245225131511688,
+ "num_tokens": 698064.0,
+ "reward": 0.4444866180419922,
+ "reward_std": 0.32400256395339966,
+ "rewards/true_env_reward_fn/mean": 0.4444866180419922,
+ "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "step": 170,
+ "step_time": 4.471538110999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 95.0,
+ "completions/mean_terminated_length": 95.0,
+ "completions/min_length": 79.0,
+ "completions/min_terminated_length": 79.0,
+ "entropy": 1.4204387068748474,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1646534949541092,
+ "kl": 2.4697198568901513e-05,
+ "learning_rate": 5.228866172636899e-07,
+ "loss": 0.02632315456867218,
+ "num_tokens": 704196.0,
+ "reward": 0.304565966129303,
+ "reward_std": 0.32997164130210876,
+ "rewards/true_env_reward_fn/mean": 0.304565966129303,
+ "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "step": 171,
+ "step_time": 5.0436168590003945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1254178285598755,
+ "epoch": 1.3983739837398375,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15058821439743042,
+ "kl": 1.8407325114822015e-05,
+ "learning_rate": 5.192791644186662e-07,
+ "loss": 0.025478817522525787,
+ "num_tokens": 706411.0,
+ "reward": 0.7279239892959595,
+ "reward_std": 0.11376125365495682,
+ "rewards/true_env_reward_fn/mean": 0.7279239892959595,
+ "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "step": 172,
+ "step_time": 3.2261944119982218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0894773602485657,
+ "epoch": 1.4065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1687694787979126,
+ "kl": 2.6046765015053097e-05,
+ "learning_rate": 5.156610312374013e-07,
+ "loss": -0.056941211223602295,
+ "num_tokens": 711212.0,
+ "reward": 0.4907146692276001,
+ "reward_std": 0.3376546800136566,
+ "rewards/true_env_reward_fn/mean": 0.4907146692276001,
+ "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "step": 173,
+ "step_time": 3.9498180619993946
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 60.125,
+ "completions/mean_terminated_length": 60.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2238691449165344,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19584441184997559,
+ "kl": 4.8285241064149886e-05,
+ "learning_rate": 5.120325416899629e-07,
+ "loss": 0.0766875222325325,
+ "num_tokens": 715409.0,
+ "reward": 0.4593355059623718,
+ "reward_std": 0.3909546136856079,
+ "rewards/true_env_reward_fn/mean": 0.4593355059623718,
+ "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "step": 174,
+ "step_time": 4.100519798999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1763202548027039,
+ "epoch": 1.4227642276422765,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1448555737733841,
+ "kl": 1.2618989785551094e-05,
+ "learning_rate": 5.08394020673734e-07,
+ "loss": -0.012558378279209137,
+ "num_tokens": 722327.0,
+ "reward": 0.15966665744781494,
+ "reward_std": 0.3235519230365753,
+ "rewards/true_env_reward_fn/mean": 0.15966665744781494,
+ "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "step": 175,
+ "step_time": 3.875348296000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2852763533592224,
+ "epoch": 1.4308943089430894,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22303813695907593,
+ "kl": 4.580334098136518e-05,
+ "learning_rate": 5.047457939843227e-07,
+ "loss": -0.09214464575052261,
+ "num_tokens": 726828.0,
+ "reward": 0.25830498337745667,
+ "reward_std": 0.37860655784606934,
+ "rewards/true_env_reward_fn/mean": 0.25830498337745667,
+ "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "step": 176,
+ "step_time": 4.241473076999682
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3609731197357178,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.001985745271668e-05,
+ "kl": 1.4942165307729738e-05,
+ "learning_rate": 5.010881882863893e-07,
+ "loss": 7.44550789022469e-07,
+ "num_tokens": 729930.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 177,
+ "step_time": 3.3902666960002534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2678966522216797,
+ "epoch": 1.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15351757407188416,
+ "kl": 1.6737100395403104e-05,
+ "learning_rate": 4.974215310843967e-07,
+ "loss": 0.041131969541311264,
+ "num_tokens": 732155.0,
+ "reward": 0.7803820371627808,
+ "reward_std": 0.08667682856321335,
+ "rewards/true_env_reward_fn/mean": 0.7803820371627808,
+ "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "step": 178,
+ "step_time": 3.637111981999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.146271526813507,
+ "epoch": 1.4552845528455285,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18466196954250336,
+ "kl": 3.719841197380447e-05,
+ "learning_rate": 4.937461506932859e-07,
+ "loss": 0.029051154851913452,
+ "num_tokens": 735418.0,
+ "reward": 0.40377071499824524,
+ "reward_std": 0.28345924615859985,
+ "rewards/true_env_reward_fn/mean": 0.40377071499824524,
+ "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "step": 179,
+ "step_time": 3.340555791999577
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.3902945518493652,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1020251139998436,
+ "kl": 1.8220500351162627e-05,
+ "learning_rate": 4.900623762090777e-07,
+ "loss": -0.002344265580177307,
+ "num_tokens": 740540.0,
+ "reward": 0.3590222895145416,
+ "reward_std": 0.12487777322530746,
+ "rewards/true_env_reward_fn/mean": 0.3590222895145416,
+ "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "step": 180,
+ "step_time": 7.219923718001155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.517557680606842,
+ "epoch": 1.4715447154471546,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24192294478416443,
+ "kl": 4.1268089262302965e-05,
+ "learning_rate": 4.863705374794055e-07,
+ "loss": 0.09132950007915497,
+ "num_tokens": 744723.0,
+ "reward": 0.23991000652313232,
+ "reward_std": 0.2837013602256775,
+ "rewards/true_env_reward_fn/mean": 0.23991000652313232,
+ "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "step": 181,
+ "step_time": 3.696339096999509
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.249614655971527,
+ "epoch": 1.4796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1566745787858963,
+ "kl": 2.6629099920683075e-05,
+ "learning_rate": 4.826709650739811e-07,
+ "loss": 0.003972277045249939,
+ "num_tokens": 748979.0,
+ "reward": 0.4935140311717987,
+ "reward_std": 0.41978561878204346,
+ "rewards/true_env_reward_fn/mean": 0.4935140311717987,
+ "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "step": 182,
+ "step_time": 3.316512920000605
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1666916608810425,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001005820304271765,
+ "kl": 1.2246940059412736e-05,
+ "learning_rate": 4.789639902549948e-07,
+ "loss": 6.330609494398232e-07,
+ "num_tokens": 751323.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 183,
+ "step_time": 3.7099916660008603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3246222138404846,
+ "epoch": 1.4959349593495934,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18140996992588043,
+ "kl": 3.1042441150930244e-05,
+ "learning_rate": 4.752499449474535e-07,
+ "loss": -0.022353097796440125,
+ "num_tokens": 755494.0,
+ "reward": 0.4561777412891388,
+ "reward_std": 0.2439236342906952,
+ "rewards/true_env_reward_fn/mean": 0.4561777412891388,
+ "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "step": 184,
+ "step_time": 3.7916486710000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 75.75,
+ "completions/mean_terminated_length": 75.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.2625707983970642,
+ "epoch": 1.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11702944338321686,
+ "kl": 1.503958355897339e-05,
+ "learning_rate": 4.715291617094607e-07,
+ "loss": 0.023916304111480713,
+ "num_tokens": 758432.0,
+ "reward": 0.5015827417373657,
+ "reward_std": 0.1783808320760727,
+ "rewards/true_env_reward_fn/mean": 0.5015827417373657,
+ "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "step": 185,
+ "step_time": 4.226409274000616
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1460023522377014,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11851620674133301,
+ "kl": 1.3728345948038623e-05,
+ "learning_rate": 4.678019737024387e-07,
+ "loss": 0.0831337422132492,
+ "num_tokens": 764336.0,
+ "reward": 0.19094166159629822,
+ "reward_std": 0.30934420228004456,
+ "rewards/true_env_reward_fn/mean": 0.19094166159629822,
+ "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "step": 186,
+ "step_time": 3.563357556000483
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1291148662567139,
+ "epoch": 1.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.864256960805506e-05,
+ "kl": 1.081683785741916e-05,
+ "learning_rate": 4.6406871466129704e-07,
+ "loss": 5.412177870312007e-07,
+ "num_tokens": 766608.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 187,
+ "step_time": 3.534869859002356
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 92.0,
+ "completions/mean_terminated_length": 92.0,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3645328283309937,
+ "epoch": 1.5284552845528454,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18168245255947113,
+ "kl": 2.6857565899263136e-05,
+ "learning_rate": 4.6032971886454956e-07,
+ "loss": -0.005156125873327255,
+ "num_tokens": 774496.0,
+ "reward": 0.11249999701976776,
+ "reward_std": 0.20856082439422607,
+ "rewards/true_env_reward_fn/mean": 0.11249999701976776,
+ "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "step": 188,
+ "step_time": 8.34005261099992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1495982110500336,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14486820995807648,
+ "kl": 1.3202762147557223e-05,
+ "learning_rate": 4.5658532110438337e-07,
+ "loss": -0.0010610297322273254,
+ "num_tokens": 777186.0,
+ "reward": 0.5879127383232117,
+ "reward_std": 0.05142820253968239,
+ "rewards/true_env_reward_fn/mean": 0.5879127383232117,
+ "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "step": 189,
+ "step_time": 3.4856022139993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.241140365600586,
+ "epoch": 1.5447154471544715,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001250067143701017,
+ "kl": 1.5482702110602986e-05,
+ "learning_rate": 4.52835856656681e-07,
+ "loss": 7.80837922320643e-07,
+ "num_tokens": 779965.0,
+ "reward": 0.6861198544502258,
+ "reward_std": 0.08807206153869629,
+ "rewards/true_env_reward_fn/mean": 0.6861198544502258,
+ "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "step": 190,
+ "step_time": 3.904181735999373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2944807410240173,
+ "epoch": 1.5528455284552845,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2030111849308014,
+ "kl": 3.585523518268019e-05,
+ "learning_rate": 4.490816612509991e-07,
+ "loss": 0.0143373291939497,
+ "num_tokens": 786140.0,
+ "reward": 0.35173332691192627,
+ "reward_std": 0.18115806579589844,
+ "rewards/true_env_reward_fn/mean": 0.35173332691192627,
+ "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "step": 191,
+ "step_time": 4.005758510000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 58.0,
+ "completions/mean_terminated_length": 58.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2177271246910095,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11188288033008575,
+ "kl": 2.3622495064046234e-05,
+ "learning_rate": 4.45323071040508e-07,
+ "loss": -0.02369789034128189,
+ "num_tokens": 790424.0,
+ "reward": 0.4888629913330078,
+ "reward_std": 0.23310808837413788,
+ "rewards/true_env_reward_fn/mean": 0.4888629913330078,
+ "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "step": 192,
+ "step_time": 3.2518814809991454
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.192966103553772,
+ "epoch": 1.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1378823071718216,
+ "kl": 1.7358055174554465e-05,
+ "learning_rate": 4.4156042257189143e-07,
+ "loss": 0.06256310641765594,
+ "num_tokens": 794521.0,
+ "reward": 0.5219699740409851,
+ "reward_std": 0.06214587390422821,
+ "rewards/true_env_reward_fn/mean": 0.5219699740409851,
+ "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "step": 193,
+ "step_time": 4.222739491999164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1770159006118774,
+ "epoch": 1.5772357723577235,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19550059735774994,
+ "kl": 2.4871268578863237e-05,
+ "learning_rate": 4.377940527552125e-07,
+ "loss": 0.05841376632452011,
+ "num_tokens": 798194.0,
+ "reward": 0.42302167415618896,
+ "reward_std": 0.2911272346973419,
+ "rewards/true_env_reward_fn/mean": 0.42302167415618896,
+ "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "step": 194,
+ "step_time": 3.8170270639984665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.5244255661964417,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1872202306985855,
+ "kl": 2.5990483663917985e-05,
+ "learning_rate": 4.340242988337462e-07,
+ "loss": -0.044112429022789,
+ "num_tokens": 802802.0,
+ "reward": 0.33745431900024414,
+ "reward_std": 0.22955451905727386,
+ "rewards/true_env_reward_fn/mean": 0.33745431900024414,
+ "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "step": 195,
+ "step_time": 4.27381555500142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.345891296863556,
+ "epoch": 1.5934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16891054809093475,
+ "kl": 2.7261638024356216e-05,
+ "learning_rate": 4.3025149835378275e-07,
+ "loss": -0.139386385679245,
+ "num_tokens": 807881.0,
+ "reward": 0.3240283131599426,
+ "reward_std": 0.2803676128387451,
+ "rewards/true_env_reward_fn/mean": 0.3240283131599426,
+ "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "step": 196,
+ "step_time": 4.981287381999209
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.246802031993866,
+ "epoch": 1.6016260162601625,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2096729576587677,
+ "kl": 3.5958016269432846e-05,
+ "learning_rate": 4.2647598913440264e-07,
+ "loss": -0.02941281348466873,
+ "num_tokens": 812500.0,
+ "reward": 0.4126526415348053,
+ "reward_std": 0.36393746733665466,
+ "rewards/true_env_reward_fn/mean": 0.4126526415348053,
+ "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "step": 197,
+ "step_time": 3.101726017999681
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4369062185287476,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1862105429172516,
+ "kl": 4.7646244638599455e-05,
+ "learning_rate": 4.2269810923722965e-07,
+ "loss": 0.0521145761013031,
+ "num_tokens": 818718.0,
+ "reward": 0.26869943737983704,
+ "reward_std": 0.1483483463525772,
+ "rewards/true_env_reward_fn/mean": 0.26869943737983704,
+ "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "step": 198,
+ "step_time": 4.287780451000799
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.192937195301056,
+ "epoch": 1.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2013663798570633,
+ "kl": 1.8760739294521045e-05,
+ "learning_rate": 4.189181969361588e-07,
+ "loss": 0.07236722111701965,
+ "num_tokens": 825728.0,
+ "reward": 0.23110000789165497,
+ "reward_std": 0.23212090134620667,
+ "rewards/true_env_reward_fn/mean": 0.23110000789165497,
+ "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "step": 199,
+ "step_time": 4.708717262998107
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.3792839050292969,
+ "epoch": 1.6260162601626016,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.7985117463395e-05,
+ "kl": 1.4280476534622721e-05,
+ "learning_rate": 4.1513659068706814e-07,
+ "loss": 7.153485626076872e-07,
+ "num_tokens": 830318.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 200,
+ "step_time": 3.9055351140013954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0505937337875366,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13188514113426208,
+ "kl": 2.842090088961413e-05,
+ "learning_rate": 4.1135362909751326e-07,
+ "loss": -0.0017508119344711304,
+ "num_tokens": 834933.0,
+ "reward": 0.36545002460479736,
+ "reward_std": 0.24526984989643097,
+ "rewards/true_env_reward_fn/mean": 0.36545002460479736,
+ "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "step": 201,
+ "step_time": 3.895525625997834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.3750707507133484,
+ "epoch": 1.6422764227642277,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1798955649137497,
+ "kl": 2.5428611479583196e-05,
+ "learning_rate": 4.075696508964076e-07,
+ "loss": 0.1843666434288025,
+ "num_tokens": 838076.0,
+ "reward": 0.44641831517219543,
+ "reward_std": 0.30040720105171204,
+ "rewards/true_env_reward_fn/mean": 0.44641831517219543,
+ "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "step": 202,
+ "step_time": 6.74463491500137
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 79.125,
+ "completions/mean_terminated_length": 79.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3369249105453491,
+ "epoch": 1.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11249764263629913,
+ "kl": 1.1453126717242412e-05,
+ "learning_rate": 4.0378499490369267e-07,
+ "loss": -0.08210685849189758,
+ "num_tokens": 841933.0,
+ "reward": 0.3737962245941162,
+ "reward_std": 0.13184049725532532,
+ "rewards/true_env_reward_fn/mean": 0.3737962245941162,
+ "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "step": 203,
+ "step_time": 5.5639925510004105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.5737199783325195,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19394491612911224,
+ "kl": 5.05317857459886e-05,
+ "learning_rate": 4e-07,
+ "loss": 0.05622926354408264,
+ "num_tokens": 847711.0,
+ "reward": 0.10725000500679016,
+ "reward_std": 0.19980257749557495,
+ "rewards/true_env_reward_fn/mean": 0.10725000500679016,
+ "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "step": 204,
+ "step_time": 4.918089437000162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.119917094707489,
+ "epoch": 1.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10553985089063644,
+ "kl": 1.2793303994840244e-05,
+ "learning_rate": 3.9621500509630725e-07,
+ "loss": -0.011355768889188766,
+ "num_tokens": 849519.0,
+ "reward": 0.6593211889266968,
+ "reward_std": 0.11862105131149292,
+ "rewards/true_env_reward_fn/mean": 0.6593211889266968,
+ "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "step": 205,
+ "step_time": 4.127652793999005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 75.125,
+ "completions/mean_terminated_length": 75.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3856809735298157,
+ "epoch": 1.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1911258101463318,
+ "kl": 4.2569914512569085e-05,
+ "learning_rate": 3.9243034910359247e-07,
+ "loss": 0.10561336576938629,
+ "num_tokens": 854156.0,
+ "reward": 0.2516202926635742,
+ "reward_std": 0.22357939183712006,
+ "rewards/true_env_reward_fn/mean": 0.2516202926635742,
+ "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "step": 206,
+ "step_time": 4.714620994000143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.4012945890426636,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226125180721283,
+ "kl": 5.4918069963605376e-05,
+ "learning_rate": 3.886463709024868e-07,
+ "loss": 0.012949362397193909,
+ "num_tokens": 856151.0,
+ "reward": 0.6204804182052612,
+ "reward_std": 0.44673967361450195,
+ "rewards/true_env_reward_fn/mean": 0.6204804182052612,
+ "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "step": 207,
+ "step_time": 4.094810713999323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3708943128585815,
+ "epoch": 1.6910569105691056,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12903675436973572,
+ "kl": 1.4145812656352064e-05,
+ "learning_rate": 3.8486340931293187e-07,
+ "loss": -0.07838249206542969,
+ "num_tokens": 859380.0,
+ "reward": 0.6400156021118164,
+ "reward_std": 0.1022576317191124,
+ "rewards/true_env_reward_fn/mean": 0.6400156021118164,
+ "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "step": 208,
+ "step_time": 4.297900428997309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.75,
+ "completions/mean_terminated_length": 56.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2782961130142212,
+ "epoch": 1.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12437833100557327,
+ "kl": 1.4473939245362999e-05,
+ "learning_rate": 3.8108180306384135e-07,
+ "loss": -0.036324724555015564,
+ "num_tokens": 862714.0,
+ "reward": 0.43419933319091797,
+ "reward_std": 0.15345513820648193,
+ "rewards/true_env_reward_fn/mean": 0.43419933319091797,
+ "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "step": 209,
+ "step_time": 3.47861851900052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 123.875,
+ "completions/mean_terminated_length": 68.42857360839844,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.5170292258262634,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1543467938899994,
+ "kl": 2.3121557205740828e-05,
+ "learning_rate": 3.7730189076277037e-07,
+ "loss": -0.3810324966907501,
+ "num_tokens": 869789.0,
+ "reward": 0.3795333504676819,
+ "reward_std": 0.13815106451511383,
+ "rewards/true_env_reward_fn/mean": 0.3795333504676819,
+ "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "step": 210,
+ "step_time": 20.698896928999602
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2324861884117126,
+ "epoch": 1.7154471544715446,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11762262135744095,
+ "kl": 1.9743249595194357e-05,
+ "learning_rate": 3.735240108655973e-07,
+ "loss": -0.09104303270578384,
+ "num_tokens": 872604.0,
+ "reward": 0.5916227102279663,
+ "reward_std": 0.1752101182937622,
+ "rewards/true_env_reward_fn/mean": 0.5916227102279663,
+ "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "step": 211,
+ "step_time": 5.218213289999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.4448966979980469,
+ "epoch": 1.7235772357723578,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1924143135547638,
+ "kl": 3.186432604707079e-05,
+ "learning_rate": 3.697485016462174e-07,
+ "loss": 0.025449808686971664,
+ "num_tokens": 877921.0,
+ "reward": 0.1028124988079071,
+ "reward_std": 0.2254277467727661,
+ "rewards/true_env_reward_fn/mean": 0.1028124988079071,
+ "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "step": 212,
+ "step_time": 3.415528882000217
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.185910701751709,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001214127623825334,
+ "kl": 1.3199866316426778e-05,
+ "learning_rate": 3.659757011662538e-07,
+ "loss": 6.776077725589857e-07,
+ "num_tokens": 880344.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 213,
+ "step_time": 3.5156538789997285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 90.75,
+ "completions/mean_terminated_length": 90.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2697569727897644,
+ "epoch": 1.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1553589552640915,
+ "kl": 1.621047795197228e-05,
+ "learning_rate": 3.622059472447875e-07,
+ "loss": -0.111361563205719,
+ "num_tokens": 885006.0,
+ "reward": 0.5490846633911133,
+ "reward_std": 0.14710450172424316,
+ "rewards/true_env_reward_fn/mean": 0.5490846633911133,
+ "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "step": 214,
+ "step_time": 7.051423932000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1904898285865784,
+ "epoch": 1.7479674796747968,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19009968638420105,
+ "kl": 1.6167180092452327e-05,
+ "learning_rate": 3.5843957742810864e-07,
+ "loss": -0.006048411130905151,
+ "num_tokens": 888255.0,
+ "reward": 0.5893601179122925,
+ "reward_std": 0.16021940112113953,
+ "rewards/true_env_reward_fn/mean": 0.5893601179122925,
+ "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "step": 215,
+ "step_time": 3.0270869319992926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2750649452209473,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1521027386188507,
+ "kl": 4.317680577514693e-05,
+ "learning_rate": 3.5467692895949205e-07,
+ "loss": -0.04247616231441498,
+ "num_tokens": 892490.0,
+ "reward": 0.1841849982738495,
+ "reward_std": 0.30015870928764343,
+ "rewards/true_env_reward_fn/mean": 0.1841849982738495,
+ "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "step": 216,
+ "step_time": 3.461749838998003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2730335593223572,
+ "epoch": 1.7642276422764227,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11831019073724747,
+ "kl": 2.1095927877468057e-05,
+ "learning_rate": 3.509183387490009e-07,
+ "loss": -0.051231447607278824,
+ "num_tokens": 894688.0,
+ "reward": 0.49520131945610046,
+ "reward_std": 0.3804744482040405,
+ "rewards/true_env_reward_fn/mean": 0.49520131945610046,
+ "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "step": 217,
+ "step_time": 3.7184635590019752
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 47.75,
+ "completions/mean_terminated_length": 47.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3015827536582947,
+ "epoch": 1.7723577235772359,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21724702417850494,
+ "kl": 3.562447545846226e-05,
+ "learning_rate": 3.471641433433191e-07,
+ "loss": -0.028775859624147415,
+ "num_tokens": 899066.0,
+ "reward": 0.3376166820526123,
+ "reward_std": 0.21976198256015778,
+ "rewards/true_env_reward_fn/mean": 0.3376166820526123,
+ "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "step": 218,
+ "step_time": 3.344433074000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1390373706817627,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001077545020962134,
+ "kl": 1.554161144667887e-05,
+ "learning_rate": 3.434146788956166e-07,
+ "loss": 7.676237032683275e-07,
+ "num_tokens": 902263.0,
+ "reward": 0.5682899951934814,
+ "reward_std": 0.1217179074883461,
+ "rewards/true_env_reward_fn/mean": 0.5682899951934814,
+ "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "step": 219,
+ "step_time": 3.0424018219982827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1991714239120483,
+ "epoch": 1.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11428863555192947,
+ "kl": 1.1265870853094384e-05,
+ "learning_rate": 3.3967028113545045e-07,
+ "loss": -0.02927359938621521,
+ "num_tokens": 906206.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 220,
+ "step_time": 3.5531271640011255
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2546668648719788,
+ "epoch": 1.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1504630595445633,
+ "kl": 1.5663241811125772e-05,
+ "learning_rate": 3.3593128533870314e-07,
+ "loss": 0.04806854575872421,
+ "num_tokens": 909149.0,
+ "reward": 0.49599751830101013,
+ "reward_std": 0.15429075062274933,
+ "rewards/true_env_reward_fn/mean": 0.49599751830101013,
+ "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "step": 221,
+ "step_time": 3.1345955030010373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1428714394569397,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.310600969707593e-05,
+ "kl": 1.1706195891747484e-05,
+ "learning_rate": 3.321980262975613e-07,
+ "loss": 5.957842290627013e-07,
+ "num_tokens": 914211.0,
+ "reward": 0.3258306384086609,
+ "reward_std": 0.4338511824607849,
+ "rewards/true_env_reward_fn/mean": 0.3258306384086609,
+ "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "step": 222,
+ "step_time": 3.8445859539988305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1629019975662231,
+ "epoch": 1.8130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010947151895379648,
+ "kl": 1.3530024261854123e-05,
+ "learning_rate": 3.2847083829053923e-07,
+ "loss": 6.723923888785066e-07,
+ "num_tokens": 916850.0,
+ "reward": 0.5956059694290161,
+ "reward_std": 0.09251586347818375,
+ "rewards/true_env_reward_fn/mean": 0.5956059694290161,
+ "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "step": 223,
+ "step_time": 3.3021794950000185
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1237311363220215,
+ "epoch": 1.821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14958374202251434,
+ "kl": 4.083753810846247e-05,
+ "learning_rate": 3.2475005505254657e-07,
+ "loss": 0.09789139032363892,
+ "num_tokens": 919901.0,
+ "reward": 0.4179220199584961,
+ "reward_std": 0.2486819326877594,
+ "rewards/true_env_reward_fn/mean": 0.4179220199584961,
+ "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "step": 224,
+ "step_time": 4.954825423001239
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 52.375,
+ "completions/mean_terminated_length": 52.375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1394256949424744,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1480269879102707,
+ "kl": 3.2915593692450784e-05,
+ "learning_rate": 3.210360097450052e-07,
+ "loss": 0.11319300532341003,
+ "num_tokens": 923840.0,
+ "reward": 0.4335233271121979,
+ "reward_std": 0.31114333868026733,
+ "rewards/true_env_reward_fn/mean": 0.4335233271121979,
+ "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "step": 225,
+ "step_time": 3.777259659000265
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.354669451713562,
+ "epoch": 1.8373983739837398,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13672730326652527,
+ "kl": 2.2323275516100693e-05,
+ "learning_rate": 3.173290349260188e-07,
+ "loss": -0.0521523654460907,
+ "num_tokens": 927882.0,
+ "reward": 0.57341468334198,
+ "reward_std": 0.11576741933822632,
+ "rewards/true_env_reward_fn/mean": 0.57341468334198,
+ "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "step": 226,
+ "step_time": 3.613498073998926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1750767230987549,
+ "epoch": 1.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.230674147605896,
+ "kl": 1.800864629331045e-05,
+ "learning_rate": 3.136294625205945e-07,
+ "loss": 0.058730173856019974,
+ "num_tokens": 930225.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 227,
+ "step_time": 3.3819083769976714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 40.875,
+ "completions/mean_terminated_length": 40.875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2825847864151,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.30391862988471985,
+ "kl": 2.0106223928451072e-05,
+ "learning_rate": 3.0993762379092235e-07,
+ "loss": -0.10232458263635635,
+ "num_tokens": 933048.0,
+ "reward": 0.45667415857315063,
+ "reward_std": 0.26290765404701233,
+ "rewards/true_env_reward_fn/mean": 0.45667415857315063,
+ "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "step": 228,
+ "step_time": 2.698590726000475
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 41.125,
+ "completions/mean_terminated_length": 41.125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3104369640350342,
+ "epoch": 1.8617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.28224605321884155,
+ "kl": 6.227439189387951e-05,
+ "learning_rate": 3.06253849306714e-07,
+ "loss": -0.008679002523422241,
+ "num_tokens": 938217.0,
+ "reward": 0.2567799985408783,
+ "reward_std": 0.2555168867111206,
+ "rewards/true_env_reward_fn/mean": 0.2567799985408783,
+ "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "step": 229,
+ "step_time": 2.7451177700022527
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0164751410484314,
+ "epoch": 1.8699186991869918,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011273886048002169,
+ "kl": 1.1985231139988173e-05,
+ "learning_rate": 3.0257846891560323e-07,
+ "loss": 5.987301960885816e-07,
+ "num_tokens": 941049.0,
+ "reward": 0.6971603631973267,
+ "reward_std": 0.2594861686229706,
+ "rewards/true_env_reward_fn/mean": 0.6971603631973267,
+ "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "step": 230,
+ "step_time": 2.917641182999432
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.1991845965385437,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015363915008492768,
+ "kl": 1.4284144981502322e-05,
+ "learning_rate": 2.989118117136107e-07,
+ "loss": 6.798551339670666e-07,
+ "num_tokens": 945403.0,
+ "reward": 0.31745320558547974,
+ "reward_std": 0.42489534616470337,
+ "rewards/true_env_reward_fn/mean": 0.31745320558547974,
+ "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "step": 231,
+ "step_time": 5.5585464220002905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2020843029022217,
+ "epoch": 1.886178861788618,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13684362173080444,
+ "kl": 3.651866154541494e-05,
+ "learning_rate": 2.952542060156773e-07,
+ "loss": 0.025869816541671753,
+ "num_tokens": 949126.0,
+ "reward": 0.3984018564224243,
+ "reward_std": 0.4202974736690521,
+ "rewards/true_env_reward_fn/mean": 0.3984018564224243,
+ "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "step": 232,
+ "step_time": 3.4383463869999105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 58.625,
+ "completions/mean_terminated_length": 58.625,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2224581837654114,
+ "epoch": 1.8943089430894309,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20249564945697784,
+ "kl": 3.106597978330683e-05,
+ "learning_rate": 2.9160597932626605e-07,
+ "loss": -0.003915777429938316,
+ "num_tokens": 952815.0,
+ "reward": 0.4141089916229248,
+ "reward_std": 0.3064958453178406,
+ "rewards/true_env_reward_fn/mean": 0.4141089916229248,
+ "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "step": 233,
+ "step_time": 3.6410487339999236
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1212781071662903,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14502985775470734,
+ "kl": 3.0683338081871625e-05,
+ "learning_rate": 2.879674583100372e-07,
+ "loss": -0.0860406681895256,
+ "num_tokens": 958656.0,
+ "reward": 0.30326664447784424,
+ "reward_std": 0.24800051748752594,
+ "rewards/true_env_reward_fn/mean": 0.30326664447784424,
+ "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "step": 234,
+ "step_time": 3.8798253620007017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 60.75,
+ "completions/mean_terminated_length": 60.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.425286054611206,
+ "epoch": 1.910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2035023272037506,
+ "kl": 5.512987627298571e-05,
+ "learning_rate": 2.843389687625986e-07,
+ "loss": -0.010439477860927582,
+ "num_tokens": 962638.0,
+ "reward": 0.3157375156879425,
+ "reward_std": 0.46047845482826233,
+ "rewards/true_env_reward_fn/mean": 0.3157375156879425,
+ "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "step": 235,
+ "step_time": 4.114513064998391
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 77.625,
+ "completions/mean_terminated_length": 77.625,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4385854601860046,
+ "epoch": 1.91869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12982772290706635,
+ "kl": 1.2591926861205138e-05,
+ "learning_rate": 2.807208355813339e-07,
+ "loss": 0.09093751758337021,
+ "num_tokens": 965755.0,
+ "reward": 0.6140732765197754,
+ "reward_std": 0.27462607622146606,
+ "rewards/true_env_reward_fn/mean": 0.6140732765197754,
+ "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "step": 236,
+ "step_time": 5.147667763001664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.3476852178573608,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14480431377887726,
+ "kl": 3.8014684832887724e-05,
+ "learning_rate": 2.771133827363101e-07,
+ "loss": -0.07322391867637634,
+ "num_tokens": 970918.0,
+ "reward": 0.3427826166152954,
+ "reward_std": 0.42430612444877625,
+ "rewards/true_env_reward_fn/mean": 0.3427826166152954,
+ "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "step": 237,
+ "step_time": 4.67846887900123
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 221.0,
+ "completions/max_terminated_length": 221.0,
+ "completions/mean_length": 94.25,
+ "completions/mean_terminated_length": 94.25,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3279914855957031,
+ "epoch": 1.934959349593496,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10826019197702408,
+ "kl": 1.3074863090878353e-05,
+ "learning_rate": 2.7351693324127037e-07,
+ "loss": -0.05556309223175049,
+ "num_tokens": 975120.0,
+ "reward": 0.64573073387146,
+ "reward_std": 0.22739914059638977,
+ "rewards/true_env_reward_fn/mean": 0.64573073387146,
+ "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "step": 238,
+ "step_time": 9.220254810001279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3695034384727478,
+ "epoch": 1.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12166028469800949,
+ "kl": 2.6563114261080045e-05,
+ "learning_rate": 2.6993180912471055e-07,
+ "loss": -0.05334407091140747,
+ "num_tokens": 980254.0,
+ "reward": 0.37203267216682434,
+ "reward_std": 0.20089927315711975,
+ "rewards/true_env_reward_fn/mean": 0.37203267216682434,
+ "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "step": 239,
+ "step_time": 4.224964968001586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.75,
+ "completions/mean_terminated_length": 53.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.247464120388031,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2346416860818863,
+ "kl": 5.1520751185307745e-05,
+ "learning_rate": 2.6635833140104405e-07,
+ "loss": -0.0905834436416626,
+ "num_tokens": 983244.0,
+ "reward": 0.575507640838623,
+ "reward_std": 0.17391785979270935,
+ "rewards/true_env_reward_fn/mean": 0.575507640838623,
+ "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "step": 240,
+ "step_time": 3.357481237999309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2175387144088745,
+ "epoch": 1.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1870066374540329,
+ "kl": 2.2185965462995227e-05,
+ "learning_rate": 2.6279682004185894e-07,
+ "loss": -0.07365687191486359,
+ "num_tokens": 985574.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 241,
+ "step_time": 4.527591582998866
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2727615237236023,
+ "epoch": 1.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.226071804529056e-05,
+ "kl": 1.4014385214977665e-05,
+ "learning_rate": 2.592475939472668e-07,
+ "loss": 7.015369192231447e-07,
+ "num_tokens": 988868.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 242,
+ "step_time": 4.238274277000528
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1519948840141296,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12447996437549591,
+ "kl": 4.700180943473242e-05,
+ "learning_rate": 2.557109709173482e-07,
+ "loss": 0.08819369971752167,
+ "num_tokens": 994068.0,
+ "reward": 0.3696666657924652,
+ "reward_std": 0.18488828837871552,
+ "rewards/true_env_reward_fn/mean": 0.3696666657924652,
+ "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "step": 243,
+ "step_time": 3.3367313500002638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.25,
+ "completions/mean_terminated_length": 68.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0537148416042328,
+ "epoch": 1.9837398373983741,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17681685090065002,
+ "kl": 3.6240851841284893e-05,
+ "learning_rate": 2.521872676236972e-07,
+ "loss": 0.05281040072441101,
+ "num_tokens": 1000650.0,
+ "reward": 0.12209999561309814,
+ "reward_std": 0.2502918243408203,
+ "rewards/true_env_reward_fn/mean": 0.12209999561309814,
+ "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "step": 244,
+ "step_time": 5.2957401019993995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 61.0,
+ "completions/min_terminated_length": 61.0,
+ "entropy": 1.3126497864723206,
+ "epoch": 1.9918699186991868,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16036106646060944,
+ "kl": 6.061139720259234e-05,
+ "learning_rate": 2.48676799581066e-07,
+ "loss": 0.09418506920337677,
+ "num_tokens": 1005737.0,
+ "reward": 0.31175702810287476,
+ "reward_std": 0.38867074251174927,
+ "rewards/true_env_reward_fn/mean": 0.31175702810287476,
+ "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "step": 245,
+ "step_time": 6.259088058999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 48.5,
+ "completions/mean_terminated_length": 48.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.21333646774292,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23997871577739716,
+ "kl": 3.1378609492094256e-05,
+ "learning_rate": 2.4517988111911313e-07,
+ "loss": 0.010592922568321228,
+ "num_tokens": 1010869.0,
+ "reward": 0.33381664752960205,
+ "reward_std": 0.18213039636611938,
+ "rewards/true_env_reward_fn/mean": 0.33381664752960205,
+ "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "step": 246,
+ "step_time": 3.111915630997828
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.3962982892990112,
+ "epoch": 2.008130081300813,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21355785429477692,
+ "kl": 3.789625407080166e-05,
+ "learning_rate": 2.4169682535425927e-07,
+ "loss": 0.025682777166366577,
+ "num_tokens": 1014876.0,
+ "reward": 0.35749268531799316,
+ "reward_std": 0.29738906025886536,
+ "rewards/true_env_reward_fn/mean": 0.35749268531799316,
+ "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "step": 247,
+ "step_time": 3.35338095500083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 73.875,
+ "completions/mean_terminated_length": 73.875,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.463137686252594,
+ "epoch": 2.016260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18188992142677307,
+ "kl": 1.746804719005013e-05,
+ "learning_rate": 2.382279441616492e-07,
+ "loss": -0.17857304215431213,
+ "num_tokens": 1018383.0,
+ "reward": 0.5329012274742126,
+ "reward_std": 0.055823445320129395,
+ "rewards/true_env_reward_fn/mean": 0.5329012274742126,
+ "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "step": 248,
+ "step_time": 5.210386754000865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 79.25,
+ "completions/mean_terminated_length": 79.25,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.4478936195373535,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.744662434561178e-05,
+ "kl": 1.3336490155779757e-05,
+ "learning_rate": 2.3477354814722762e-07,
+ "loss": 6.725406365148956e-07,
+ "num_tokens": 1022753.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 249,
+ "step_time": 8.798317029002646
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.37166029214859,
+ "epoch": 2.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1542455554008484,
+ "kl": 2.0379054603836266e-05,
+ "learning_rate": 2.313339466199264e-07,
+ "loss": -0.037539318203926086,
+ "num_tokens": 1025971.0,
+ "reward": 0.6065863966941833,
+ "reward_std": 0.032470256090164185,
+ "rewards/true_env_reward_fn/mean": 0.6065863966941833,
+ "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "step": 250,
+ "step_time": 4.096263454999644
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 1025971,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-250/training_args.bin b/checkpoint-250/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-250/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-300/README.md b/checkpoint-300/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-300/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-300/adapter_config.json b/checkpoint-300/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-300/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-300/adapter_model.safetensors b/checkpoint-300/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a7bb69e4aa5c7ec9e22ddcdaf0425bdc84f8ef92
--- /dev/null
+++ b/checkpoint-300/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6714687556a2dc7226afce3096d21be7cf908f496df4aa4046e603946cdae6c
+size 8731128
diff --git a/checkpoint-300/chat_template.jinja b/checkpoint-300/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-300/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-300/optimizer.pt b/checkpoint-300/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..000c07ad367673f21a7b94af8924af5916ddc9ca
--- /dev/null
+++ b/checkpoint-300/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c6d03de482df3140c859a74e70a8a08cb87820c32024dc3ce2c6802304d1ae1
+size 17526842
diff --git a/checkpoint-300/ref/adapter_config.json b/checkpoint-300/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-300/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-300/ref/adapter_model.safetensors b/checkpoint-300/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-300/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-300/rng_state.pth b/checkpoint-300/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..46479067c27a2a4c2fdd57b24455ba6dad5457b6
--- /dev/null
+++ b/checkpoint-300/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c41259f0bbb68ce4b0c21dedacd5a801851021c681838c318a21744e1672f1
+size 14244
diff --git a/checkpoint-300/scaler.pt b/checkpoint-300/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d60b64ea1e2a0ba2c3efcf307a54af9e13f032b
--- /dev/null
+++ b/checkpoint-300/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92873c8c89778fe11b8eeb338a181eefdf056f2f8096c36bf259c3fd791afb34
+size 988
diff --git a/checkpoint-300/scheduler.pt b/checkpoint-300/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8596d3e88a3afc73532e7d6466cf60a9f6b0d26c
--- /dev/null
+++ b/checkpoint-300/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:129a2d7bcd9a56630a9f55dcea61d36958bdd2c2a73a893c6fc4f9afcdcbff14
+size 1064
diff --git a/checkpoint-300/tokenizer.json b/checkpoint-300/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-300/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-300/tokenizer_config.json b/checkpoint-300/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-300/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-300/trainer_state.json b/checkpoint-300/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..51c137478586de02ac26433714bc8fefa956f171
--- /dev/null
+++ b/checkpoint-300/trainer_state.json
@@ -0,0 +1,8434 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.4390243902439024,
+ "eval_steps": 500,
+ "global_step": 300,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1402934789657593,
+ "epoch": 1.2276422764227641,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010795716661959887,
+ "kl": 1.4652535810455447e-05,
+ "learning_rate": 5.923033215939834e-07,
+ "loss": 7.542968205598299e-07,
+ "num_tokens": 621009.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 151,
+ "step_time": 3.926544339999964
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.190350890159607,
+ "epoch": 1.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19853363931179047,
+ "kl": 2.269768037876929e-05,
+ "learning_rate": 5.889758273918683e-07,
+ "loss": 0.044217392802238464,
+ "num_tokens": 623994.0,
+ "reward": 0.4411996603012085,
+ "reward_std": 0.2517909109592438,
+ "rewards/true_env_reward_fn/mean": 0.4411996603012085,
+ "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "step": 152,
+ "step_time": 3.7339736520007136
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2316884994506836,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.35867181699723e-05,
+ "kl": 1.1579370038816705e-05,
+ "learning_rate": 5.856314121676467e-07,
+ "loss": 5.79387460675207e-07,
+ "num_tokens": 628224.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 153,
+ "step_time": 4.375236807001784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4028943181037903,
+ "epoch": 1.2520325203252032,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25418931245803833,
+ "kl": 3.6890452065563295e-05,
+ "learning_rate": 5.822703753824966e-07,
+ "loss": 0.25599968433380127,
+ "num_tokens": 631183.0,
+ "reward": 0.38683533668518066,
+ "reward_std": 0.43613559007644653,
+ "rewards/true_env_reward_fn/mean": 0.38683533668518066,
+ "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "step": 154,
+ "step_time": 6.055355972999678
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.367663562297821,
+ "epoch": 1.2601626016260163,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331811130046844,
+ "kl": 1.2863993106293492e-05,
+ "learning_rate": 5.788930179859024e-07,
+ "loss": -0.0047044456005096436,
+ "num_tokens": 636230.0,
+ "reward": 0.3489508628845215,
+ "reward_std": 0.13627417385578156,
+ "rewards/true_env_reward_fn/mean": 0.3489508628845215,
+ "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "step": 155,
+ "step_time": 3.9174396130001696
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 68.5,
+ "completions/mean_terminated_length": 68.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2759611010551453,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12738144397735596,
+ "kl": 1.5844128029129934e-05,
+ "learning_rate": 5.754996423887061e-07,
+ "loss": -0.011055335402488708,
+ "num_tokens": 640262.0,
+ "reward": 0.344404935836792,
+ "reward_std": 0.16542991995811462,
+ "rewards/true_env_reward_fn/mean": 0.344404935836792,
+ "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "step": 156,
+ "step_time": 5.706334413998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0904476642608643,
+ "epoch": 1.2764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.733699021628127e-05,
+ "kl": 1.2238857834745431e-05,
+ "learning_rate": 5.720905524360308e-07,
+ "loss": 6.076299996493617e-07,
+ "num_tokens": 645091.0,
+ "reward": 0.4731999933719635,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4731999933719635,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 157,
+ "step_time": 3.7696847109982627
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.0,
+ "completions/mean_terminated_length": 60.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3856677412986755,
+ "epoch": 1.2845528455284554,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18670028448104858,
+ "kl": 1.8415606064081658e-05,
+ "learning_rate": 5.686660533800736e-07,
+ "loss": -0.07078710198402405,
+ "num_tokens": 648179.0,
+ "reward": 0.537517786026001,
+ "reward_std": 0.1451217085123062,
+ "rewards/true_env_reward_fn/mean": 0.537517786026001,
+ "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "step": 158,
+ "step_time": 3.7075291149994882
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 68.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.121916651725769,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11415883898735046,
+ "kl": 1.909901220642496e-05,
+ "learning_rate": 5.652264518527725e-07,
+ "loss": -0.04401372745633125,
+ "num_tokens": 652044.0,
+ "reward": 0.5182899832725525,
+ "reward_std": 0.21869486570358276,
+ "rewards/true_env_reward_fn/mean": 0.5182899832725525,
+ "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "step": 159,
+ "step_time": 3.8929355969994504
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5205118060112,
+ "epoch": 1.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2403375506401062,
+ "kl": 3.7574073758150917e-05,
+ "learning_rate": 5.617720558383508e-07,
+ "loss": 0.26385918259620667,
+ "num_tokens": 656362.0,
+ "reward": 0.41201668977737427,
+ "reward_std": 0.2023741453886032,
+ "rewards/true_env_reward_fn/mean": 0.41201668977737427,
+ "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "step": 160,
+ "step_time": 6.157555950998358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3556928038597107,
+ "epoch": 1.3089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18135924637317657,
+ "kl": 2.6372636057203636e-05,
+ "learning_rate": 5.583031746457407e-07,
+ "loss": -0.10538280755281448,
+ "num_tokens": 659977.0,
+ "reward": 0.4239906072616577,
+ "reward_std": 0.3287450969219208,
+ "rewards/true_env_reward_fn/mean": 0.4239906072616577,
+ "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "step": 161,
+ "step_time": 4.709477423999488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 52.5,
+ "completions/mean_terminated_length": 52.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 0.9743769466876984,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.178797647356987,
+ "kl": 1.2532927030406427e-05,
+ "learning_rate": 5.548201188808869e-07,
+ "loss": -0.04164513945579529,
+ "num_tokens": 661409.0,
+ "reward": 0.8041956424713135,
+ "reward_std": 0.1363772451877594,
+ "rewards/true_env_reward_fn/mean": 0.8041956424713135,
+ "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "step": 162,
+ "step_time": 3.1791253910014348
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.3072250485420227,
+ "epoch": 1.3252032520325203,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16768279671669006,
+ "kl": 1.89386219062726e-05,
+ "learning_rate": 5.513232004189339e-07,
+ "loss": -0.01292814314365387,
+ "num_tokens": 666504.0,
+ "reward": 0.27981066703796387,
+ "reward_std": 0.2949208915233612,
+ "rewards/true_env_reward_fn/mean": 0.27981066703796387,
+ "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "step": 163,
+ "step_time": 3.778431355000066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 51.0,
+ "completions/mean_terminated_length": 51.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1984856128692627,
+ "epoch": 1.3333333333333333,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13123764097690582,
+ "kl": 1.9091786271019373e-05,
+ "learning_rate": 5.478127323763027e-07,
+ "loss": 0.035523779690265656,
+ "num_tokens": 671004.0,
+ "reward": 0.2771500051021576,
+ "reward_std": 0.30146247148513794,
+ "rewards/true_env_reward_fn/mean": 0.2771500051021576,
+ "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "step": 164,
+ "step_time": 3.304021460000513
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 84.875,
+ "completions/mean_terminated_length": 84.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2726752758026123,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15594874322414398,
+ "kl": 2.5981638827943243e-05,
+ "learning_rate": 5.442890290826518e-07,
+ "loss": -0.01398652046918869,
+ "num_tokens": 677307.0,
+ "reward": 0.35573017597198486,
+ "reward_std": 0.25944042205810547,
+ "rewards/true_env_reward_fn/mean": 0.35573017597198486,
+ "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "step": 165,
+ "step_time": 4.972808451999299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 103.625,
+ "completions/mean_terminated_length": 103.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0949090719223022,
+ "epoch": 1.3495934959349594,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07191785424947739,
+ "kl": 1.4828182884230046e-05,
+ "learning_rate": 5.407524060527332e-07,
+ "loss": -0.05063021928071976,
+ "num_tokens": 682076.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 166,
+ "step_time": 8.769379133000257
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1857684254646301,
+ "epoch": 1.3577235772357723,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.923542191041633e-05,
+ "kl": 1.0165251751459436e-05,
+ "learning_rate": 5.37203179958141e-07,
+ "loss": 5.114516170579009e-07,
+ "num_tokens": 685500.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 167,
+ "step_time": 3.1906087530005607
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 74.25,
+ "completions/mean_terminated_length": 74.25,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.527149498462677,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13164250552654266,
+ "kl": 1.8541333702160046e-05,
+ "learning_rate": 5.33641668598956e-07,
+ "loss": -0.2347300797700882,
+ "num_tokens": 688318.0,
+ "reward": 0.7218117713928223,
+ "reward_std": 0.1818692982196808,
+ "rewards/true_env_reward_fn/mean": 0.7218117713928223,
+ "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "step": 168,
+ "step_time": 8.50137474999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 63.0,
+ "completions/min_terminated_length": 63.0,
+ "entropy": 1.3749513030052185,
+ "epoch": 1.3739837398373984,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13246971368789673,
+ "kl": 1.2620409506780561e-05,
+ "learning_rate": 5.300681908752895e-07,
+ "loss": 0.024534843862056732,
+ "num_tokens": 692541.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 169,
+ "step_time": 3.9512340759993094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2032299041748047,
+ "epoch": 1.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11185794323682785,
+ "kl": 1.5517784049734473e-05,
+ "learning_rate": 5.264830667587295e-07,
+ "loss": -0.05245225131511688,
+ "num_tokens": 698064.0,
+ "reward": 0.4444866180419922,
+ "reward_std": 0.32400256395339966,
+ "rewards/true_env_reward_fn/mean": 0.4444866180419922,
+ "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "step": 170,
+ "step_time": 4.471538110999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 95.0,
+ "completions/mean_terminated_length": 95.0,
+ "completions/min_length": 79.0,
+ "completions/min_terminated_length": 79.0,
+ "entropy": 1.4204387068748474,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1646534949541092,
+ "kl": 2.4697198568901513e-05,
+ "learning_rate": 5.228866172636899e-07,
+ "loss": 0.02632315456867218,
+ "num_tokens": 704196.0,
+ "reward": 0.304565966129303,
+ "reward_std": 0.32997164130210876,
+ "rewards/true_env_reward_fn/mean": 0.304565966129303,
+ "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "step": 171,
+ "step_time": 5.0436168590003945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1254178285598755,
+ "epoch": 1.3983739837398375,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15058821439743042,
+ "kl": 1.8407325114822015e-05,
+ "learning_rate": 5.192791644186662e-07,
+ "loss": 0.025478817522525787,
+ "num_tokens": 706411.0,
+ "reward": 0.7279239892959595,
+ "reward_std": 0.11376125365495682,
+ "rewards/true_env_reward_fn/mean": 0.7279239892959595,
+ "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "step": 172,
+ "step_time": 3.2261944119982218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0894773602485657,
+ "epoch": 1.4065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1687694787979126,
+ "kl": 2.6046765015053097e-05,
+ "learning_rate": 5.156610312374013e-07,
+ "loss": -0.056941211223602295,
+ "num_tokens": 711212.0,
+ "reward": 0.4907146692276001,
+ "reward_std": 0.3376546800136566,
+ "rewards/true_env_reward_fn/mean": 0.4907146692276001,
+ "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "step": 173,
+ "step_time": 3.9498180619993946
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 60.125,
+ "completions/mean_terminated_length": 60.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2238691449165344,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19584441184997559,
+ "kl": 4.8285241064149886e-05,
+ "learning_rate": 5.120325416899629e-07,
+ "loss": 0.0766875222325325,
+ "num_tokens": 715409.0,
+ "reward": 0.4593355059623718,
+ "reward_std": 0.3909546136856079,
+ "rewards/true_env_reward_fn/mean": 0.4593355059623718,
+ "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "step": 174,
+ "step_time": 4.100519798999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1763202548027039,
+ "epoch": 1.4227642276422765,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1448555737733841,
+ "kl": 1.2618989785551094e-05,
+ "learning_rate": 5.08394020673734e-07,
+ "loss": -0.012558378279209137,
+ "num_tokens": 722327.0,
+ "reward": 0.15966665744781494,
+ "reward_std": 0.3235519230365753,
+ "rewards/true_env_reward_fn/mean": 0.15966665744781494,
+ "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "step": 175,
+ "step_time": 3.875348296000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2852763533592224,
+ "epoch": 1.4308943089430894,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22303813695907593,
+ "kl": 4.580334098136518e-05,
+ "learning_rate": 5.047457939843227e-07,
+ "loss": -0.09214464575052261,
+ "num_tokens": 726828.0,
+ "reward": 0.25830498337745667,
+ "reward_std": 0.37860655784606934,
+ "rewards/true_env_reward_fn/mean": 0.25830498337745667,
+ "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "step": 176,
+ "step_time": 4.241473076999682
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3609731197357178,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.001985745271668e-05,
+ "kl": 1.4942165307729738e-05,
+ "learning_rate": 5.010881882863893e-07,
+ "loss": 7.44550789022469e-07,
+ "num_tokens": 729930.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 177,
+ "step_time": 3.3902666960002534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2678966522216797,
+ "epoch": 1.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15351757407188416,
+ "kl": 1.6737100395403104e-05,
+ "learning_rate": 4.974215310843967e-07,
+ "loss": 0.041131969541311264,
+ "num_tokens": 732155.0,
+ "reward": 0.7803820371627808,
+ "reward_std": 0.08667682856321335,
+ "rewards/true_env_reward_fn/mean": 0.7803820371627808,
+ "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "step": 178,
+ "step_time": 3.637111981999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.146271526813507,
+ "epoch": 1.4552845528455285,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18466196954250336,
+ "kl": 3.719841197380447e-05,
+ "learning_rate": 4.937461506932859e-07,
+ "loss": 0.029051154851913452,
+ "num_tokens": 735418.0,
+ "reward": 0.40377071499824524,
+ "reward_std": 0.28345924615859985,
+ "rewards/true_env_reward_fn/mean": 0.40377071499824524,
+ "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "step": 179,
+ "step_time": 3.340555791999577
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.3902945518493652,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1020251139998436,
+ "kl": 1.8220500351162627e-05,
+ "learning_rate": 4.900623762090777e-07,
+ "loss": -0.002344265580177307,
+ "num_tokens": 740540.0,
+ "reward": 0.3590222895145416,
+ "reward_std": 0.12487777322530746,
+ "rewards/true_env_reward_fn/mean": 0.3590222895145416,
+ "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "step": 180,
+ "step_time": 7.219923718001155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.517557680606842,
+ "epoch": 1.4715447154471546,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24192294478416443,
+ "kl": 4.1268089262302965e-05,
+ "learning_rate": 4.863705374794055e-07,
+ "loss": 0.09132950007915497,
+ "num_tokens": 744723.0,
+ "reward": 0.23991000652313232,
+ "reward_std": 0.2837013602256775,
+ "rewards/true_env_reward_fn/mean": 0.23991000652313232,
+ "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "step": 181,
+ "step_time": 3.696339096999509
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.249614655971527,
+ "epoch": 1.4796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1566745787858963,
+ "kl": 2.6629099920683075e-05,
+ "learning_rate": 4.826709650739811e-07,
+ "loss": 0.003972277045249939,
+ "num_tokens": 748979.0,
+ "reward": 0.4935140311717987,
+ "reward_std": 0.41978561878204346,
+ "rewards/true_env_reward_fn/mean": 0.4935140311717987,
+ "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "step": 182,
+ "step_time": 3.316512920000605
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1666916608810425,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001005820304271765,
+ "kl": 1.2246940059412736e-05,
+ "learning_rate": 4.789639902549948e-07,
+ "loss": 6.330609494398232e-07,
+ "num_tokens": 751323.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 183,
+ "step_time": 3.7099916660008603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3246222138404846,
+ "epoch": 1.4959349593495934,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18140996992588043,
+ "kl": 3.1042441150930244e-05,
+ "learning_rate": 4.752499449474535e-07,
+ "loss": -0.022353097796440125,
+ "num_tokens": 755494.0,
+ "reward": 0.4561777412891388,
+ "reward_std": 0.2439236342906952,
+ "rewards/true_env_reward_fn/mean": 0.4561777412891388,
+ "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "step": 184,
+ "step_time": 3.7916486710000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 75.75,
+ "completions/mean_terminated_length": 75.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.2625707983970642,
+ "epoch": 1.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11702944338321686,
+ "kl": 1.503958355897339e-05,
+ "learning_rate": 4.715291617094607e-07,
+ "loss": 0.023916304111480713,
+ "num_tokens": 758432.0,
+ "reward": 0.5015827417373657,
+ "reward_std": 0.1783808320760727,
+ "rewards/true_env_reward_fn/mean": 0.5015827417373657,
+ "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "step": 185,
+ "step_time": 4.226409274000616
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1460023522377014,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11851620674133301,
+ "kl": 1.3728345948038623e-05,
+ "learning_rate": 4.678019737024387e-07,
+ "loss": 0.0831337422132492,
+ "num_tokens": 764336.0,
+ "reward": 0.19094166159629822,
+ "reward_std": 0.30934420228004456,
+ "rewards/true_env_reward_fn/mean": 0.19094166159629822,
+ "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "step": 186,
+ "step_time": 3.563357556000483
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1291148662567139,
+ "epoch": 1.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.864256960805506e-05,
+ "kl": 1.081683785741916e-05,
+ "learning_rate": 4.6406871466129704e-07,
+ "loss": 5.412177870312007e-07,
+ "num_tokens": 766608.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 187,
+ "step_time": 3.534869859002356
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 92.0,
+ "completions/mean_terminated_length": 92.0,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3645328283309937,
+ "epoch": 1.5284552845528454,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18168245255947113,
+ "kl": 2.6857565899263136e-05,
+ "learning_rate": 4.6032971886454956e-07,
+ "loss": -0.005156125873327255,
+ "num_tokens": 774496.0,
+ "reward": 0.11249999701976776,
+ "reward_std": 0.20856082439422607,
+ "rewards/true_env_reward_fn/mean": 0.11249999701976776,
+ "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "step": 188,
+ "step_time": 8.34005261099992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1495982110500336,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14486820995807648,
+ "kl": 1.3202762147557223e-05,
+ "learning_rate": 4.5658532110438337e-07,
+ "loss": -0.0010610297322273254,
+ "num_tokens": 777186.0,
+ "reward": 0.5879127383232117,
+ "reward_std": 0.05142820253968239,
+ "rewards/true_env_reward_fn/mean": 0.5879127383232117,
+ "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "step": 189,
+ "step_time": 3.4856022139993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.241140365600586,
+ "epoch": 1.5447154471544715,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001250067143701017,
+ "kl": 1.5482702110602986e-05,
+ "learning_rate": 4.52835856656681e-07,
+ "loss": 7.80837922320643e-07,
+ "num_tokens": 779965.0,
+ "reward": 0.6861198544502258,
+ "reward_std": 0.08807206153869629,
+ "rewards/true_env_reward_fn/mean": 0.6861198544502258,
+ "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "step": 190,
+ "step_time": 3.904181735999373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2944807410240173,
+ "epoch": 1.5528455284552845,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2030111849308014,
+ "kl": 3.585523518268019e-05,
+ "learning_rate": 4.490816612509991e-07,
+ "loss": 0.0143373291939497,
+ "num_tokens": 786140.0,
+ "reward": 0.35173332691192627,
+ "reward_std": 0.18115806579589844,
+ "rewards/true_env_reward_fn/mean": 0.35173332691192627,
+ "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "step": 191,
+ "step_time": 4.005758510000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 58.0,
+ "completions/mean_terminated_length": 58.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2177271246910095,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11188288033008575,
+ "kl": 2.3622495064046234e-05,
+ "learning_rate": 4.45323071040508e-07,
+ "loss": -0.02369789034128189,
+ "num_tokens": 790424.0,
+ "reward": 0.4888629913330078,
+ "reward_std": 0.23310808837413788,
+ "rewards/true_env_reward_fn/mean": 0.4888629913330078,
+ "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "step": 192,
+ "step_time": 3.2518814809991454
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.192966103553772,
+ "epoch": 1.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1378823071718216,
+ "kl": 1.7358055174554465e-05,
+ "learning_rate": 4.4156042257189143e-07,
+ "loss": 0.06256310641765594,
+ "num_tokens": 794521.0,
+ "reward": 0.5219699740409851,
+ "reward_std": 0.06214587390422821,
+ "rewards/true_env_reward_fn/mean": 0.5219699740409851,
+ "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "step": 193,
+ "step_time": 4.222739491999164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1770159006118774,
+ "epoch": 1.5772357723577235,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19550059735774994,
+ "kl": 2.4871268578863237e-05,
+ "learning_rate": 4.377940527552125e-07,
+ "loss": 0.05841376632452011,
+ "num_tokens": 798194.0,
+ "reward": 0.42302167415618896,
+ "reward_std": 0.2911272346973419,
+ "rewards/true_env_reward_fn/mean": 0.42302167415618896,
+ "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "step": 194,
+ "step_time": 3.8170270639984665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.5244255661964417,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1872202306985855,
+ "kl": 2.5990483663917985e-05,
+ "learning_rate": 4.340242988337462e-07,
+ "loss": -0.044112429022789,
+ "num_tokens": 802802.0,
+ "reward": 0.33745431900024414,
+ "reward_std": 0.22955451905727386,
+ "rewards/true_env_reward_fn/mean": 0.33745431900024414,
+ "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "step": 195,
+ "step_time": 4.27381555500142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.345891296863556,
+ "epoch": 1.5934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16891054809093475,
+ "kl": 2.7261638024356216e-05,
+ "learning_rate": 4.3025149835378275e-07,
+ "loss": -0.139386385679245,
+ "num_tokens": 807881.0,
+ "reward": 0.3240283131599426,
+ "reward_std": 0.2803676128387451,
+ "rewards/true_env_reward_fn/mean": 0.3240283131599426,
+ "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "step": 196,
+ "step_time": 4.981287381999209
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.246802031993866,
+ "epoch": 1.6016260162601625,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2096729576587677,
+ "kl": 3.5958016269432846e-05,
+ "learning_rate": 4.2647598913440264e-07,
+ "loss": -0.02941281348466873,
+ "num_tokens": 812500.0,
+ "reward": 0.4126526415348053,
+ "reward_std": 0.36393746733665466,
+ "rewards/true_env_reward_fn/mean": 0.4126526415348053,
+ "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "step": 197,
+ "step_time": 3.101726017999681
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4369062185287476,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1862105429172516,
+ "kl": 4.7646244638599455e-05,
+ "learning_rate": 4.2269810923722965e-07,
+ "loss": 0.0521145761013031,
+ "num_tokens": 818718.0,
+ "reward": 0.26869943737983704,
+ "reward_std": 0.1483483463525772,
+ "rewards/true_env_reward_fn/mean": 0.26869943737983704,
+ "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "step": 198,
+ "step_time": 4.287780451000799
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.192937195301056,
+ "epoch": 1.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2013663798570633,
+ "kl": 1.8760739294521045e-05,
+ "learning_rate": 4.189181969361588e-07,
+ "loss": 0.07236722111701965,
+ "num_tokens": 825728.0,
+ "reward": 0.23110000789165497,
+ "reward_std": 0.23212090134620667,
+ "rewards/true_env_reward_fn/mean": 0.23110000789165497,
+ "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "step": 199,
+ "step_time": 4.708717262998107
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.3792839050292969,
+ "epoch": 1.6260162601626016,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.7985117463395e-05,
+ "kl": 1.4280476534622721e-05,
+ "learning_rate": 4.1513659068706814e-07,
+ "loss": 7.153485626076872e-07,
+ "num_tokens": 830318.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 200,
+ "step_time": 3.9055351140013954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0505937337875366,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13188514113426208,
+ "kl": 2.842090088961413e-05,
+ "learning_rate": 4.1135362909751326e-07,
+ "loss": -0.0017508119344711304,
+ "num_tokens": 834933.0,
+ "reward": 0.36545002460479736,
+ "reward_std": 0.24526984989643097,
+ "rewards/true_env_reward_fn/mean": 0.36545002460479736,
+ "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "step": 201,
+ "step_time": 3.895525625997834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.3750707507133484,
+ "epoch": 1.6422764227642277,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1798955649137497,
+ "kl": 2.5428611479583196e-05,
+ "learning_rate": 4.075696508964076e-07,
+ "loss": 0.1843666434288025,
+ "num_tokens": 838076.0,
+ "reward": 0.44641831517219543,
+ "reward_std": 0.30040720105171204,
+ "rewards/true_env_reward_fn/mean": 0.44641831517219543,
+ "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "step": 202,
+ "step_time": 6.74463491500137
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 79.125,
+ "completions/mean_terminated_length": 79.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3369249105453491,
+ "epoch": 1.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11249764263629913,
+ "kl": 1.1453126717242412e-05,
+ "learning_rate": 4.0378499490369267e-07,
+ "loss": -0.08210685849189758,
+ "num_tokens": 841933.0,
+ "reward": 0.3737962245941162,
+ "reward_std": 0.13184049725532532,
+ "rewards/true_env_reward_fn/mean": 0.3737962245941162,
+ "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "step": 203,
+ "step_time": 5.5639925510004105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.5737199783325195,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19394491612911224,
+ "kl": 5.05317857459886e-05,
+ "learning_rate": 4e-07,
+ "loss": 0.05622926354408264,
+ "num_tokens": 847711.0,
+ "reward": 0.10725000500679016,
+ "reward_std": 0.19980257749557495,
+ "rewards/true_env_reward_fn/mean": 0.10725000500679016,
+ "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "step": 204,
+ "step_time": 4.918089437000162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.119917094707489,
+ "epoch": 1.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10553985089063644,
+ "kl": 1.2793303994840244e-05,
+ "learning_rate": 3.9621500509630725e-07,
+ "loss": -0.011355768889188766,
+ "num_tokens": 849519.0,
+ "reward": 0.6593211889266968,
+ "reward_std": 0.11862105131149292,
+ "rewards/true_env_reward_fn/mean": 0.6593211889266968,
+ "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "step": 205,
+ "step_time": 4.127652793999005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 75.125,
+ "completions/mean_terminated_length": 75.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3856809735298157,
+ "epoch": 1.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1911258101463318,
+ "kl": 4.2569914512569085e-05,
+ "learning_rate": 3.9243034910359247e-07,
+ "loss": 0.10561336576938629,
+ "num_tokens": 854156.0,
+ "reward": 0.2516202926635742,
+ "reward_std": 0.22357939183712006,
+ "rewards/true_env_reward_fn/mean": 0.2516202926635742,
+ "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "step": 206,
+ "step_time": 4.714620994000143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.4012945890426636,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226125180721283,
+ "kl": 5.4918069963605376e-05,
+ "learning_rate": 3.886463709024868e-07,
+ "loss": 0.012949362397193909,
+ "num_tokens": 856151.0,
+ "reward": 0.6204804182052612,
+ "reward_std": 0.44673967361450195,
+ "rewards/true_env_reward_fn/mean": 0.6204804182052612,
+ "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "step": 207,
+ "step_time": 4.094810713999323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3708943128585815,
+ "epoch": 1.6910569105691056,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12903675436973572,
+ "kl": 1.4145812656352064e-05,
+ "learning_rate": 3.8486340931293187e-07,
+ "loss": -0.07838249206542969,
+ "num_tokens": 859380.0,
+ "reward": 0.6400156021118164,
+ "reward_std": 0.1022576317191124,
+ "rewards/true_env_reward_fn/mean": 0.6400156021118164,
+ "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "step": 208,
+ "step_time": 4.297900428997309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.75,
+ "completions/mean_terminated_length": 56.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2782961130142212,
+ "epoch": 1.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12437833100557327,
+ "kl": 1.4473939245362999e-05,
+ "learning_rate": 3.8108180306384135e-07,
+ "loss": -0.036324724555015564,
+ "num_tokens": 862714.0,
+ "reward": 0.43419933319091797,
+ "reward_std": 0.15345513820648193,
+ "rewards/true_env_reward_fn/mean": 0.43419933319091797,
+ "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "step": 209,
+ "step_time": 3.47861851900052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 123.875,
+ "completions/mean_terminated_length": 68.42857360839844,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.5170292258262634,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1543467938899994,
+ "kl": 2.3121557205740828e-05,
+ "learning_rate": 3.7730189076277037e-07,
+ "loss": -0.3810324966907501,
+ "num_tokens": 869789.0,
+ "reward": 0.3795333504676819,
+ "reward_std": 0.13815106451511383,
+ "rewards/true_env_reward_fn/mean": 0.3795333504676819,
+ "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "step": 210,
+ "step_time": 20.698896928999602
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2324861884117126,
+ "epoch": 1.7154471544715446,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11762262135744095,
+ "kl": 1.9743249595194357e-05,
+ "learning_rate": 3.735240108655973e-07,
+ "loss": -0.09104303270578384,
+ "num_tokens": 872604.0,
+ "reward": 0.5916227102279663,
+ "reward_std": 0.1752101182937622,
+ "rewards/true_env_reward_fn/mean": 0.5916227102279663,
+ "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "step": 211,
+ "step_time": 5.218213289999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.4448966979980469,
+ "epoch": 1.7235772357723578,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1924143135547638,
+ "kl": 3.186432604707079e-05,
+ "learning_rate": 3.697485016462174e-07,
+ "loss": 0.025449808686971664,
+ "num_tokens": 877921.0,
+ "reward": 0.1028124988079071,
+ "reward_std": 0.2254277467727661,
+ "rewards/true_env_reward_fn/mean": 0.1028124988079071,
+ "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "step": 212,
+ "step_time": 3.415528882000217
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.185910701751709,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001214127623825334,
+ "kl": 1.3199866316426778e-05,
+ "learning_rate": 3.659757011662538e-07,
+ "loss": 6.776077725589857e-07,
+ "num_tokens": 880344.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 213,
+ "step_time": 3.5156538789997285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 90.75,
+ "completions/mean_terminated_length": 90.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2697569727897644,
+ "epoch": 1.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1553589552640915,
+ "kl": 1.621047795197228e-05,
+ "learning_rate": 3.622059472447875e-07,
+ "loss": -0.111361563205719,
+ "num_tokens": 885006.0,
+ "reward": 0.5490846633911133,
+ "reward_std": 0.14710450172424316,
+ "rewards/true_env_reward_fn/mean": 0.5490846633911133,
+ "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "step": 214,
+ "step_time": 7.051423932000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1904898285865784,
+ "epoch": 1.7479674796747968,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19009968638420105,
+ "kl": 1.6167180092452327e-05,
+ "learning_rate": 3.5843957742810864e-07,
+ "loss": -0.006048411130905151,
+ "num_tokens": 888255.0,
+ "reward": 0.5893601179122925,
+ "reward_std": 0.16021940112113953,
+ "rewards/true_env_reward_fn/mean": 0.5893601179122925,
+ "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "step": 215,
+ "step_time": 3.0270869319992926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2750649452209473,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1521027386188507,
+ "kl": 4.317680577514693e-05,
+ "learning_rate": 3.5467692895949205e-07,
+ "loss": -0.04247616231441498,
+ "num_tokens": 892490.0,
+ "reward": 0.1841849982738495,
+ "reward_std": 0.30015870928764343,
+ "rewards/true_env_reward_fn/mean": 0.1841849982738495,
+ "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "step": 216,
+ "step_time": 3.461749838998003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2730335593223572,
+ "epoch": 1.7642276422764227,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11831019073724747,
+ "kl": 2.1095927877468057e-05,
+ "learning_rate": 3.509183387490009e-07,
+ "loss": -0.051231447607278824,
+ "num_tokens": 894688.0,
+ "reward": 0.49520131945610046,
+ "reward_std": 0.3804744482040405,
+ "rewards/true_env_reward_fn/mean": 0.49520131945610046,
+ "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "step": 217,
+ "step_time": 3.7184635590019752
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 47.75,
+ "completions/mean_terminated_length": 47.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3015827536582947,
+ "epoch": 1.7723577235772359,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21724702417850494,
+ "kl": 3.562447545846226e-05,
+ "learning_rate": 3.471641433433191e-07,
+ "loss": -0.028775859624147415,
+ "num_tokens": 899066.0,
+ "reward": 0.3376166820526123,
+ "reward_std": 0.21976198256015778,
+ "rewards/true_env_reward_fn/mean": 0.3376166820526123,
+ "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "step": 218,
+ "step_time": 3.344433074000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1390373706817627,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001077545020962134,
+ "kl": 1.554161144667887e-05,
+ "learning_rate": 3.434146788956166e-07,
+ "loss": 7.676237032683275e-07,
+ "num_tokens": 902263.0,
+ "reward": 0.5682899951934814,
+ "reward_std": 0.1217179074883461,
+ "rewards/true_env_reward_fn/mean": 0.5682899951934814,
+ "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "step": 219,
+ "step_time": 3.0424018219982827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1991714239120483,
+ "epoch": 1.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11428863555192947,
+ "kl": 1.1265870853094384e-05,
+ "learning_rate": 3.3967028113545045e-07,
+ "loss": -0.02927359938621521,
+ "num_tokens": 906206.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 220,
+ "step_time": 3.5531271640011255
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2546668648719788,
+ "epoch": 1.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1504630595445633,
+ "kl": 1.5663241811125772e-05,
+ "learning_rate": 3.3593128533870314e-07,
+ "loss": 0.04806854575872421,
+ "num_tokens": 909149.0,
+ "reward": 0.49599751830101013,
+ "reward_std": 0.15429075062274933,
+ "rewards/true_env_reward_fn/mean": 0.49599751830101013,
+ "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "step": 221,
+ "step_time": 3.1345955030010373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1428714394569397,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.310600969707593e-05,
+ "kl": 1.1706195891747484e-05,
+ "learning_rate": 3.321980262975613e-07,
+ "loss": 5.957842290627013e-07,
+ "num_tokens": 914211.0,
+ "reward": 0.3258306384086609,
+ "reward_std": 0.4338511824607849,
+ "rewards/true_env_reward_fn/mean": 0.3258306384086609,
+ "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "step": 222,
+ "step_time": 3.8445859539988305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1629019975662231,
+ "epoch": 1.8130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010947151895379648,
+ "kl": 1.3530024261854123e-05,
+ "learning_rate": 3.2847083829053923e-07,
+ "loss": 6.723923888785066e-07,
+ "num_tokens": 916850.0,
+ "reward": 0.5956059694290161,
+ "reward_std": 0.09251586347818375,
+ "rewards/true_env_reward_fn/mean": 0.5956059694290161,
+ "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "step": 223,
+ "step_time": 3.3021794950000185
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1237311363220215,
+ "epoch": 1.821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14958374202251434,
+ "kl": 4.083753810846247e-05,
+ "learning_rate": 3.2475005505254657e-07,
+ "loss": 0.09789139032363892,
+ "num_tokens": 919901.0,
+ "reward": 0.4179220199584961,
+ "reward_std": 0.2486819326877594,
+ "rewards/true_env_reward_fn/mean": 0.4179220199584961,
+ "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "step": 224,
+ "step_time": 4.954825423001239
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 52.375,
+ "completions/mean_terminated_length": 52.375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1394256949424744,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1480269879102707,
+ "kl": 3.2915593692450784e-05,
+ "learning_rate": 3.210360097450052e-07,
+ "loss": 0.11319300532341003,
+ "num_tokens": 923840.0,
+ "reward": 0.4335233271121979,
+ "reward_std": 0.31114333868026733,
+ "rewards/true_env_reward_fn/mean": 0.4335233271121979,
+ "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "step": 225,
+ "step_time": 3.777259659000265
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.354669451713562,
+ "epoch": 1.8373983739837398,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13672730326652527,
+ "kl": 2.2323275516100693e-05,
+ "learning_rate": 3.173290349260188e-07,
+ "loss": -0.0521523654460907,
+ "num_tokens": 927882.0,
+ "reward": 0.57341468334198,
+ "reward_std": 0.11576741933822632,
+ "rewards/true_env_reward_fn/mean": 0.57341468334198,
+ "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "step": 226,
+ "step_time": 3.613498073998926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1750767230987549,
+ "epoch": 1.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.230674147605896,
+ "kl": 1.800864629331045e-05,
+ "learning_rate": 3.136294625205945e-07,
+ "loss": 0.058730173856019974,
+ "num_tokens": 930225.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 227,
+ "step_time": 3.3819083769976714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 40.875,
+ "completions/mean_terminated_length": 40.875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2825847864151,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.30391862988471985,
+ "kl": 2.0106223928451072e-05,
+ "learning_rate": 3.0993762379092235e-07,
+ "loss": -0.10232458263635635,
+ "num_tokens": 933048.0,
+ "reward": 0.45667415857315063,
+ "reward_std": 0.26290765404701233,
+ "rewards/true_env_reward_fn/mean": 0.45667415857315063,
+ "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "step": 228,
+ "step_time": 2.698590726000475
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 41.125,
+ "completions/mean_terminated_length": 41.125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3104369640350342,
+ "epoch": 1.8617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.28224605321884155,
+ "kl": 6.227439189387951e-05,
+ "learning_rate": 3.06253849306714e-07,
+ "loss": -0.008679002523422241,
+ "num_tokens": 938217.0,
+ "reward": 0.2567799985408783,
+ "reward_std": 0.2555168867111206,
+ "rewards/true_env_reward_fn/mean": 0.2567799985408783,
+ "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "step": 229,
+ "step_time": 2.7451177700022527
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0164751410484314,
+ "epoch": 1.8699186991869918,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011273886048002169,
+ "kl": 1.1985231139988173e-05,
+ "learning_rate": 3.0257846891560323e-07,
+ "loss": 5.987301960885816e-07,
+ "num_tokens": 941049.0,
+ "reward": 0.6971603631973267,
+ "reward_std": 0.2594861686229706,
+ "rewards/true_env_reward_fn/mean": 0.6971603631973267,
+ "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "step": 230,
+ "step_time": 2.917641182999432
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.1991845965385437,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015363915008492768,
+ "kl": 1.4284144981502322e-05,
+ "learning_rate": 2.989118117136107e-07,
+ "loss": 6.798551339670666e-07,
+ "num_tokens": 945403.0,
+ "reward": 0.31745320558547974,
+ "reward_std": 0.42489534616470337,
+ "rewards/true_env_reward_fn/mean": 0.31745320558547974,
+ "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "step": 231,
+ "step_time": 5.5585464220002905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2020843029022217,
+ "epoch": 1.886178861788618,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13684362173080444,
+ "kl": 3.651866154541494e-05,
+ "learning_rate": 2.952542060156773e-07,
+ "loss": 0.025869816541671753,
+ "num_tokens": 949126.0,
+ "reward": 0.3984018564224243,
+ "reward_std": 0.4202974736690521,
+ "rewards/true_env_reward_fn/mean": 0.3984018564224243,
+ "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "step": 232,
+ "step_time": 3.4383463869999105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 58.625,
+ "completions/mean_terminated_length": 58.625,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2224581837654114,
+ "epoch": 1.8943089430894309,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20249564945697784,
+ "kl": 3.106597978330683e-05,
+ "learning_rate": 2.9160597932626605e-07,
+ "loss": -0.003915777429938316,
+ "num_tokens": 952815.0,
+ "reward": 0.4141089916229248,
+ "reward_std": 0.3064958453178406,
+ "rewards/true_env_reward_fn/mean": 0.4141089916229248,
+ "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "step": 233,
+ "step_time": 3.6410487339999236
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1212781071662903,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14502985775470734,
+ "kl": 3.0683338081871625e-05,
+ "learning_rate": 2.879674583100372e-07,
+ "loss": -0.0860406681895256,
+ "num_tokens": 958656.0,
+ "reward": 0.30326664447784424,
+ "reward_std": 0.24800051748752594,
+ "rewards/true_env_reward_fn/mean": 0.30326664447784424,
+ "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "step": 234,
+ "step_time": 3.8798253620007017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 60.75,
+ "completions/mean_terminated_length": 60.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.425286054611206,
+ "epoch": 1.910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2035023272037506,
+ "kl": 5.512987627298571e-05,
+ "learning_rate": 2.843389687625986e-07,
+ "loss": -0.010439477860927582,
+ "num_tokens": 962638.0,
+ "reward": 0.3157375156879425,
+ "reward_std": 0.46047845482826233,
+ "rewards/true_env_reward_fn/mean": 0.3157375156879425,
+ "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "step": 235,
+ "step_time": 4.114513064998391
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 77.625,
+ "completions/mean_terminated_length": 77.625,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4385854601860046,
+ "epoch": 1.91869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12982772290706635,
+ "kl": 1.2591926861205138e-05,
+ "learning_rate": 2.807208355813339e-07,
+ "loss": 0.09093751758337021,
+ "num_tokens": 965755.0,
+ "reward": 0.6140732765197754,
+ "reward_std": 0.27462607622146606,
+ "rewards/true_env_reward_fn/mean": 0.6140732765197754,
+ "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "step": 236,
+ "step_time": 5.147667763001664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.3476852178573608,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14480431377887726,
+ "kl": 3.8014684832887724e-05,
+ "learning_rate": 2.771133827363101e-07,
+ "loss": -0.07322391867637634,
+ "num_tokens": 970918.0,
+ "reward": 0.3427826166152954,
+ "reward_std": 0.42430612444877625,
+ "rewards/true_env_reward_fn/mean": 0.3427826166152954,
+ "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "step": 237,
+ "step_time": 4.67846887900123
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 221.0,
+ "completions/max_terminated_length": 221.0,
+ "completions/mean_length": 94.25,
+ "completions/mean_terminated_length": 94.25,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3279914855957031,
+ "epoch": 1.934959349593496,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10826019197702408,
+ "kl": 1.3074863090878353e-05,
+ "learning_rate": 2.7351693324127037e-07,
+ "loss": -0.05556309223175049,
+ "num_tokens": 975120.0,
+ "reward": 0.64573073387146,
+ "reward_std": 0.22739914059638977,
+ "rewards/true_env_reward_fn/mean": 0.64573073387146,
+ "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "step": 238,
+ "step_time": 9.220254810001279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3695034384727478,
+ "epoch": 1.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12166028469800949,
+ "kl": 2.6563114261080045e-05,
+ "learning_rate": 2.6993180912471055e-07,
+ "loss": -0.05334407091140747,
+ "num_tokens": 980254.0,
+ "reward": 0.37203267216682434,
+ "reward_std": 0.20089927315711975,
+ "rewards/true_env_reward_fn/mean": 0.37203267216682434,
+ "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "step": 239,
+ "step_time": 4.224964968001586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.75,
+ "completions/mean_terminated_length": 53.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.247464120388031,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2346416860818863,
+ "kl": 5.1520751185307745e-05,
+ "learning_rate": 2.6635833140104405e-07,
+ "loss": -0.0905834436416626,
+ "num_tokens": 983244.0,
+ "reward": 0.575507640838623,
+ "reward_std": 0.17391785979270935,
+ "rewards/true_env_reward_fn/mean": 0.575507640838623,
+ "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "step": 240,
+ "step_time": 3.357481237999309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2175387144088745,
+ "epoch": 1.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1870066374540329,
+ "kl": 2.2185965462995227e-05,
+ "learning_rate": 2.6279682004185894e-07,
+ "loss": -0.07365687191486359,
+ "num_tokens": 985574.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 241,
+ "step_time": 4.527591582998866
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2727615237236023,
+ "epoch": 1.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.226071804529056e-05,
+ "kl": 1.4014385214977665e-05,
+ "learning_rate": 2.592475939472668e-07,
+ "loss": 7.015369192231447e-07,
+ "num_tokens": 988868.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 242,
+ "step_time": 4.238274277000528
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1519948840141296,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12447996437549591,
+ "kl": 4.700180943473242e-05,
+ "learning_rate": 2.557109709173482e-07,
+ "loss": 0.08819369971752167,
+ "num_tokens": 994068.0,
+ "reward": 0.3696666657924652,
+ "reward_std": 0.18488828837871552,
+ "rewards/true_env_reward_fn/mean": 0.3696666657924652,
+ "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "step": 243,
+ "step_time": 3.3367313500002638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.25,
+ "completions/mean_terminated_length": 68.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0537148416042328,
+ "epoch": 1.9837398373983741,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17681685090065002,
+ "kl": 3.6240851841284893e-05,
+ "learning_rate": 2.521872676236972e-07,
+ "loss": 0.05281040072441101,
+ "num_tokens": 1000650.0,
+ "reward": 0.12209999561309814,
+ "reward_std": 0.2502918243408203,
+ "rewards/true_env_reward_fn/mean": 0.12209999561309814,
+ "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "step": 244,
+ "step_time": 5.2957401019993995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 61.0,
+ "completions/min_terminated_length": 61.0,
+ "entropy": 1.3126497864723206,
+ "epoch": 1.9918699186991868,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16036106646060944,
+ "kl": 6.061139720259234e-05,
+ "learning_rate": 2.48676799581066e-07,
+ "loss": 0.09418506920337677,
+ "num_tokens": 1005737.0,
+ "reward": 0.31175702810287476,
+ "reward_std": 0.38867074251174927,
+ "rewards/true_env_reward_fn/mean": 0.31175702810287476,
+ "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "step": 245,
+ "step_time": 6.259088058999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 48.5,
+ "completions/mean_terminated_length": 48.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.21333646774292,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23997871577739716,
+ "kl": 3.1378609492094256e-05,
+ "learning_rate": 2.4517988111911313e-07,
+ "loss": 0.010592922568321228,
+ "num_tokens": 1010869.0,
+ "reward": 0.33381664752960205,
+ "reward_std": 0.18213039636611938,
+ "rewards/true_env_reward_fn/mean": 0.33381664752960205,
+ "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "step": 246,
+ "step_time": 3.111915630997828
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.3962982892990112,
+ "epoch": 2.008130081300813,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21355785429477692,
+ "kl": 3.789625407080166e-05,
+ "learning_rate": 2.4169682535425927e-07,
+ "loss": 0.025682777166366577,
+ "num_tokens": 1014876.0,
+ "reward": 0.35749268531799316,
+ "reward_std": 0.29738906025886536,
+ "rewards/true_env_reward_fn/mean": 0.35749268531799316,
+ "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "step": 247,
+ "step_time": 3.35338095500083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 73.875,
+ "completions/mean_terminated_length": 73.875,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.463137686252594,
+ "epoch": 2.016260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18188992142677307,
+ "kl": 1.746804719005013e-05,
+ "learning_rate": 2.382279441616492e-07,
+ "loss": -0.17857304215431213,
+ "num_tokens": 1018383.0,
+ "reward": 0.5329012274742126,
+ "reward_std": 0.055823445320129395,
+ "rewards/true_env_reward_fn/mean": 0.5329012274742126,
+ "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "step": 248,
+ "step_time": 5.210386754000865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 79.25,
+ "completions/mean_terminated_length": 79.25,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.4478936195373535,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.744662434561178e-05,
+ "kl": 1.3336490155779757e-05,
+ "learning_rate": 2.3477354814722762e-07,
+ "loss": 6.725406365148956e-07,
+ "num_tokens": 1022753.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 249,
+ "step_time": 8.798317029002646
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.37166029214859,
+ "epoch": 2.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1542455554008484,
+ "kl": 2.0379054603836266e-05,
+ "learning_rate": 2.313339466199264e-07,
+ "loss": -0.037539318203926086,
+ "num_tokens": 1025971.0,
+ "reward": 0.6065863966941833,
+ "reward_std": 0.032470256090164185,
+ "rewards/true_env_reward_fn/mean": 0.6065863966941833,
+ "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "step": 250,
+ "step_time": 4.096263454999644
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4042693972587585,
+ "epoch": 2.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.163822203874588,
+ "kl": 3.525477495713858e-05,
+ "learning_rate": 2.2790944756396916e-07,
+ "loss": 0.03408379852771759,
+ "num_tokens": 1029415.0,
+ "reward": 0.37829869985580444,
+ "reward_std": 0.2773255407810211,
+ "rewards/true_env_reward_fn/mean": 0.37829869985580444,
+ "rewards/true_env_reward_fn/std": 0.2773255407810211,
+ "step": 251,
+ "step_time": 3.9464334140011488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2268111109733582,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1961166262626648,
+ "kl": 3.528672823449597e-05,
+ "learning_rate": 2.2450035761129391e-07,
+ "loss": 0.2999379336833954,
+ "num_tokens": 1032213.0,
+ "reward": 0.6772161722183228,
+ "reward_std": 0.31218820810317993,
+ "rewards/true_env_reward_fn/mean": 0.6772161722183228,
+ "rewards/true_env_reward_fn/std": 0.31218820810317993,
+ "step": 252,
+ "step_time": 7.437029113001699
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.360656499862671,
+ "epoch": 2.0569105691056913,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24334783852100372,
+ "kl": 5.1042834456893615e-05,
+ "learning_rate": 2.2110698201409787e-07,
+ "loss": 0.05509951710700989,
+ "num_tokens": 1038598.0,
+ "reward": 0.2947666645050049,
+ "reward_std": 0.19891902804374695,
+ "rewards/true_env_reward_fn/mean": 0.2947666645050049,
+ "rewards/true_env_reward_fn/std": 0.19891902804374695,
+ "step": 253,
+ "step_time": 3.7359043900014512
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 74.75,
+ "completions/mean_terminated_length": 74.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4314632415771484,
+ "epoch": 2.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1960419863462448,
+ "kl": 6.17889963905327e-05,
+ "learning_rate": 2.1772962461750342e-07,
+ "loss": 0.0954262986779213,
+ "num_tokens": 1043268.0,
+ "reward": 0.4036714732646942,
+ "reward_std": 0.42137831449508667,
+ "rewards/true_env_reward_fn/mean": 0.4036714732646942,
+ "rewards/true_env_reward_fn/std": 0.42137834429740906,
+ "step": 254,
+ "step_time": 5.565175547999388
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0521443486213684,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.464648271910846e-05,
+ "kl": 1.2018902907584561e-05,
+ "learning_rate": 2.1436858783235338e-07,
+ "loss": 6.008343689245521e-07,
+ "num_tokens": 1046517.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 255,
+ "step_time": 4.379171047001364
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.091518223285675,
+ "epoch": 2.08130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.263061656383798e-05,
+ "kl": 1.4573892713087844e-05,
+ "learning_rate": 2.110241726081317e-07,
+ "loss": 7.304333848878741e-07,
+ "num_tokens": 1050439.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 256,
+ "step_time": 3.8379976090000127
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2301559448242188,
+ "epoch": 2.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17180238664150238,
+ "kl": 2.236898035334889e-05,
+ "learning_rate": 2.076966784060165e-07,
+ "loss": 0.023751959204673767,
+ "num_tokens": 1054578.0,
+ "reward": 0.4112047851085663,
+ "reward_std": 0.05330020561814308,
+ "rewards/true_env_reward_fn/mean": 0.4112047851085663,
+ "rewards/true_env_reward_fn/std": 0.05330020561814308,
+ "step": 257,
+ "step_time": 4.424114469000415
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 52.125,
+ "completions/mean_terminated_length": 52.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2734522223472595,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2274128645658493,
+ "kl": 5.3426387239596806e-05,
+ "learning_rate": 2.043864031720667e-07,
+ "loss": 0.05165906995534897,
+ "num_tokens": 1060579.0,
+ "reward": 0.2129499912261963,
+ "reward_std": 0.2476053088903427,
+ "rewards/true_env_reward_fn/mean": 0.2129499912261963,
+ "rewards/true_env_reward_fn/std": 0.2476053088903427,
+ "step": 258,
+ "step_time": 3.4830677139998443
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 54.375,
+ "completions/mean_terminated_length": 54.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2013150453567505,
+ "epoch": 2.105691056910569,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20796725153923035,
+ "kl": 8.34841157484334e-05,
+ "learning_rate": 2.0109364331054297e-07,
+ "loss": 0.04251064360141754,
+ "num_tokens": 1065318.0,
+ "reward": 0.2038009911775589,
+ "reward_std": 0.3393669128417969,
+ "rewards/true_env_reward_fn/mean": 0.2038009911775589,
+ "rewards/true_env_reward_fn/std": 0.3393669128417969,
+ "step": 259,
+ "step_time": 3.34712773299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3005307912826538,
+ "epoch": 2.113821138211382,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2342396229505539,
+ "kl": 2.9608699151140172e-05,
+ "learning_rate": 1.9781869365736777e-07,
+ "loss": -0.11542908847332001,
+ "num_tokens": 1068352.0,
+ "reward": 0.5755212306976318,
+ "reward_std": 0.1629202961921692,
+ "rewards/true_env_reward_fn/mean": 0.5755212306976318,
+ "rewards/true_env_reward_fn/std": 0.1629202961921692,
+ "step": 260,
+ "step_time": 3.0931306170004973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1381222009658813,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13169968128204346,
+ "kl": 1.4705466128361877e-05,
+ "learning_rate": 1.9456184745372558e-07,
+ "loss": -0.026440951973199844,
+ "num_tokens": 1070847.0,
+ "reward": 0.5704532265663147,
+ "reward_std": 0.13928835093975067,
+ "rewards/true_env_reward_fn/mean": 0.5704532265663147,
+ "rewards/true_env_reward_fn/std": 0.13928835093975067,
+ "step": 261,
+ "step_time": 3.8980969309996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.241390347480774,
+ "epoch": 2.130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010450400441186503,
+ "kl": 1.6406540453317575e-05,
+ "learning_rate": 1.9132339631980622e-07,
+ "loss": 8.202600838558283e-07,
+ "num_tokens": 1076153.0,
+ "reward": 0.41493332386016846,
+ "reward_std": 0.06228968873620033,
+ "rewards/true_env_reward_fn/mean": 0.41493332386016846,
+ "rewards/true_env_reward_fn/std": 0.06228969246149063,
+ "step": 262,
+ "step_time": 3.6601423579995753
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 81.0,
+ "completions/mean_terminated_length": 81.0,
+ "completions/min_length": 67.0,
+ "completions/min_terminated_length": 67.0,
+ "entropy": 1.1326860189437866,
+ "epoch": 2.138211382113821,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001754833065206185,
+ "kl": 1.4390577689482598e-05,
+ "learning_rate": 1.881036302286923e-07,
+ "loss": 7.2446778176527e-07,
+ "num_tokens": 1080537.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 263,
+ "step_time": 4.190891189999093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.353486955165863,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.215454563498497,
+ "kl": 6.859865607111715e-05,
+ "learning_rate": 1.8490283748039515e-07,
+ "loss": 0.0181141197681427,
+ "num_tokens": 1084056.0,
+ "reward": 0.3049938380718231,
+ "reward_std": 0.4605039656162262,
+ "rewards/true_env_reward_fn/mean": 0.3049938380718231,
+ "rewards/true_env_reward_fn/std": 0.4605039954185486,
+ "step": 264,
+ "step_time": 4.504906432999633
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.375,
+ "completions/mean_terminated_length": 70.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0113105773925781,
+ "epoch": 2.154471544715447,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10072485357522964,
+ "kl": 1.4604076568502933e-05,
+ "learning_rate": 1.8172130467604085e-07,
+ "loss": -0.041721273213624954,
+ "num_tokens": 1090171.0,
+ "reward": 0.47745320200920105,
+ "reward_std": 0.2929421067237854,
+ "rewards/true_env_reward_fn/mean": 0.47745320200920105,
+ "rewards/true_env_reward_fn/std": 0.2929421067237854,
+ "step": 265,
+ "step_time": 6.299696521999067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.375,
+ "completions/mean_terminated_length": 57.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3366597294807434,
+ "epoch": 2.16260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2297964245080948,
+ "kl": 5.677436638507061e-05,
+ "learning_rate": 1.785593166922062e-07,
+ "loss": 0.20361776649951935,
+ "num_tokens": 1094358.0,
+ "reward": 0.3835672438144684,
+ "reward_std": 0.4339357614517212,
+ "rewards/true_env_reward_fn/mean": 0.3835672438144684,
+ "rewards/true_env_reward_fn/std": 0.4339357912540436,
+ "step": 266,
+ "step_time": 4.192992550000781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4040917754173279,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1913405954837799,
+ "kl": 2.2514723241329193e-05,
+ "learning_rate": 1.7541715665541276e-07,
+ "loss": -0.12278837710618973,
+ "num_tokens": 1099685.0,
+ "reward": 0.5316476821899414,
+ "reward_std": 0.1867343932390213,
+ "rewards/true_env_reward_fn/mean": 0.5316476821899414,
+ "rewards/true_env_reward_fn/std": 0.1867344230413437,
+ "step": 267,
+ "step_time": 4.577502725998784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2707499265670776,
+ "epoch": 2.178861788617886,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1252160668373108,
+ "kl": 2.1676510186807718e-05,
+ "learning_rate": 1.722951059167737e-07,
+ "loss": 0.030697450041770935,
+ "num_tokens": 1103244.0,
+ "reward": 0.47949954867362976,
+ "reward_std": 0.10323704034090042,
+ "rewards/true_env_reward_fn/mean": 0.47949954867362976,
+ "rewards/true_env_reward_fn/std": 0.10323705524206161,
+ "step": 268,
+ "step_time": 4.3898782989999745
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3778526186943054,
+ "epoch": 2.186991869918699,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19924476742744446,
+ "kl": 5.189802323002368e-05,
+ "learning_rate": 1.6919344402680231e-07,
+ "loss": 0.03015017881989479,
+ "num_tokens": 1108318.0,
+ "reward": 0.12559716403484344,
+ "reward_std": 0.22310735285282135,
+ "rewards/true_env_reward_fn/mean": 0.12559716403484344,
+ "rewards/true_env_reward_fn/std": 0.22310735285282135,
+ "step": 269,
+ "step_time": 4.379851057999986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2916911840438843,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18381677567958832,
+ "kl": 2.593698127384414e-05,
+ "learning_rate": 1.6611244871038116e-07,
+ "loss": 0.025029506534337997,
+ "num_tokens": 1116025.0,
+ "reward": 0.05200198292732239,
+ "reward_std": 0.276480108499527,
+ "rewards/true_env_reward_fn/mean": 0.05200198292732239,
+ "rewards/true_env_reward_fn/std": 0.276480108499527,
+ "step": 270,
+ "step_time": 3.6788300769985653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 94.625,
+ "completions/mean_terminated_length": 94.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2625537514686584,
+ "epoch": 2.203252032520325,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.09628148376941681,
+ "kl": 2.636932003952097e-05,
+ "learning_rate": 1.6305239584189344e-07,
+ "loss": 0.009904414415359497,
+ "num_tokens": 1120434.0,
+ "reward": 0.5924437046051025,
+ "reward_std": 0.39917245507240295,
+ "rewards/true_env_reward_fn/mean": 0.5924437046051025,
+ "rewards/true_env_reward_fn/std": 0.39917245507240295,
+ "step": 271,
+ "step_time": 9.09279120499923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2491654753684998,
+ "epoch": 2.2113821138211383,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015083610196597874,
+ "kl": 1.4522283436235739e-05,
+ "learning_rate": 1.6001355942052182e-07,
+ "loss": 7.247089683914965e-07,
+ "num_tokens": 1122349.0,
+ "reward": 0.812765896320343,
+ "reward_std": 0.04731824994087219,
+ "rewards/true_env_reward_fn/mean": 0.812765896320343,
+ "rewards/true_env_reward_fn/std": 0.04731824994087219,
+ "step": 272,
+ "step_time": 3.7730076539992297
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1462301015853882,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00012917000276502222,
+ "kl": 1.3649782886204775e-05,
+ "learning_rate": 1.569962115457138e-07,
+ "loss": 6.695274805679219e-07,
+ "num_tokens": 1124831.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 273,
+ "step_time": 3.907510233000721
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2922418713569641,
+ "epoch": 2.227642276422764,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15563185513019562,
+ "kl": 3.604595076467376e-05,
+ "learning_rate": 1.5400062239281858e-07,
+ "loss": -0.034219659864902496,
+ "num_tokens": 1128837.0,
+ "reward": 0.46255773305892944,
+ "reward_std": 0.24893923103809357,
+ "rewards/true_env_reward_fn/mean": 0.46255773305892944,
+ "rewards/true_env_reward_fn/std": 0.24893923103809357,
+ "step": 274,
+ "step_time": 7.502110859999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2709790468215942,
+ "epoch": 2.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24958185851573944,
+ "kl": 6.180045966175385e-05,
+ "learning_rate": 1.5102706018889428e-07,
+ "loss": -0.10148808360099792,
+ "num_tokens": 1132891.0,
+ "reward": 0.4240284562110901,
+ "reward_std": 0.43752968311309814,
+ "rewards/true_env_reward_fn/mean": 0.4240284562110901,
+ "rewards/true_env_reward_fn/std": 0.43752965331077576,
+ "step": 275,
+ "step_time": 3.251475233999372
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2637454867362976,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.439944602083415e-05,
+ "kl": 1.127877567341784e-05,
+ "learning_rate": 1.4807579118869146e-07,
+ "loss": 5.64579522688291e-07,
+ "num_tokens": 1137611.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 276,
+ "step_time": 3.311975311999049
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1065265536308289,
+ "epoch": 2.252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13068579137325287,
+ "kl": 4.4293181417742744e-05,
+ "learning_rate": 1.4514707965081262e-07,
+ "loss": 0.04848391190171242,
+ "num_tokens": 1142087.0,
+ "reward": 0.2852628827095032,
+ "reward_std": 0.23009054362773895,
+ "rewards/true_env_reward_fn/mean": 0.2852628827095032,
+ "rewards/true_env_reward_fn/std": 0.23009057343006134,
+ "step": 277,
+ "step_time": 3.9521008399988204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 104.5,
+ "completions/mean_terminated_length": 104.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5374161005020142,
+ "epoch": 2.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.129844531416893,
+ "kl": 3.9368313082377426e-05,
+ "learning_rate": 1.4224118781404917e-07,
+ "loss": -0.1482687145471573,
+ "num_tokens": 1147731.0,
+ "reward": 0.43388551473617554,
+ "reward_std": 0.39525240659713745,
+ "rewards/true_env_reward_fn/mean": 0.43388551473617554,
+ "rewards/true_env_reward_fn/std": 0.39525243639945984,
+ "step": 278,
+ "step_time": 7.55689369099855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.0975646376609802,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1335010528564453,
+ "kl": 2.2682882445224095e-05,
+ "learning_rate": 1.3935837587390214e-07,
+ "loss": -0.009766265749931335,
+ "num_tokens": 1151246.0,
+ "reward": 0.37655720114707947,
+ "reward_std": 0.37258440256118774,
+ "rewards/true_env_reward_fn/mean": 0.37655720114707947,
+ "rewards/true_env_reward_fn/std": 0.37258440256118774,
+ "step": 279,
+ "step_time": 3.8792882219986495
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1846895217895508,
+ "epoch": 2.2764227642276422,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.138702854514122,
+ "kl": 2.0143882466072682e-05,
+ "learning_rate": 1.3649890195928254e-07,
+ "loss": 0.013748884201049805,
+ "num_tokens": 1153222.0,
+ "reward": 0.8218116760253906,
+ "reward_std": 0.1293545663356781,
+ "rewards/true_env_reward_fn/mean": 0.8218116760253906,
+ "rewards/true_env_reward_fn/std": 0.1293545812368393,
+ "step": 280,
+ "step_time": 3.42550413599929
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 64.0,
+ "completions/mean_terminated_length": 64.0,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3451175689697266,
+ "epoch": 2.2845528455284554,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1508527249097824,
+ "kl": 1.6968931049632374e-05,
+ "learning_rate": 1.336630221093991e-07,
+ "loss": 0.04001428931951523,
+ "num_tokens": 1157606.0,
+ "reward": 0.4994586706161499,
+ "reward_std": 0.10593737661838531,
+ "rewards/true_env_reward_fn/mean": 0.4994586706161499,
+ "rewards/true_env_reward_fn/std": 0.10593737661838531,
+ "step": 281,
+ "step_time": 3.914840199000537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2436452507972717,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12380471080541611,
+ "kl": 4.350653580331709e-05,
+ "learning_rate": 1.3085099025083245e-07,
+ "loss": -0.029160797595977783,
+ "num_tokens": 1162012.0,
+ "reward": 0.5066306591033936,
+ "reward_std": 0.28914663195610046,
+ "rewards/true_env_reward_fn/mean": 0.5066306591033936,
+ "rewards/true_env_reward_fn/std": 0.28914666175842285,
+ "step": 282,
+ "step_time": 3.9093819319987233
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.10621577501297,
+ "epoch": 2.3008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001239393459400162,
+ "kl": 1.6069413049990544e-05,
+ "learning_rate": 1.2806305817479771e-07,
+ "loss": 8.174432082341809e-07,
+ "num_tokens": 1165367.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 283,
+ "step_time": 3.8117841049988783
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 66.0,
+ "completions/mean_terminated_length": 66.0,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.382466197013855,
+ "epoch": 2.3089430894308944,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1446879506111145,
+ "kl": 3.576014205464162e-05,
+ "learning_rate": 1.2529947551459964e-07,
+ "loss": -0.04731176793575287,
+ "num_tokens": 1169243.0,
+ "reward": 0.496622234582901,
+ "reward_std": 0.17860308289527893,
+ "rewards/true_env_reward_fn/mean": 0.496622234582901,
+ "rewards/true_env_reward_fn/std": 0.17860306799411774,
+ "step": 284,
+ "step_time": 3.649135475998264
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1946157813072205,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10612626373767853,
+ "kl": 2.0257402866263874e-05,
+ "learning_rate": 1.2256048972327967e-07,
+ "loss": -0.029511645436286926,
+ "num_tokens": 1173594.0,
+ "reward": 0.5235810279846191,
+ "reward_std": 0.2810492217540741,
+ "rewards/true_env_reward_fn/mean": 0.5235810279846191,
+ "rewards/true_env_reward_fn/std": 0.2810492217540741,
+ "step": 285,
+ "step_time": 6.7011265000001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2879216074943542,
+ "epoch": 2.3252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12234477698802948,
+ "kl": 1.4488860415440286e-05,
+ "learning_rate": 1.1984634605145978e-07,
+ "loss": -0.05451745539903641,
+ "num_tokens": 1176384.0,
+ "reward": 0.6496104001998901,
+ "reward_std": 0.3368098735809326,
+ "rewards/true_env_reward_fn/mean": 0.6496104001998901,
+ "rewards/true_env_reward_fn/std": 0.3368098735809326,
+ "step": 286,
+ "step_time": 4.18911992899848
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0649144053459167,
+ "epoch": 2.3333333333333335,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1467445194721222,
+ "kl": 3.947542245441582e-05,
+ "learning_rate": 1.17157287525381e-07,
+ "loss": -0.029090911149978638,
+ "num_tokens": 1179651.0,
+ "reward": 0.4754716753959656,
+ "reward_std": 0.2500669062137604,
+ "rewards/true_env_reward_fn/mean": 0.4754716753959656,
+ "rewards/true_env_reward_fn/std": 0.2500669062137604,
+ "step": 287,
+ "step_time": 4.5810332049986755
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.002779871225357,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12533478438854218,
+ "kl": 2.3054610210238025e-05,
+ "learning_rate": 1.1449355492514437e-07,
+ "loss": -0.04500773549079895,
+ "num_tokens": 1181547.0,
+ "reward": 0.7334807515144348,
+ "reward_std": 0.12557923793792725,
+ "rewards/true_env_reward_fn/mean": 0.7334807515144348,
+ "rewards/true_env_reward_fn/std": 0.12557923793792725,
+ "step": 288,
+ "step_time": 3.8661079009998502
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 81.25,
+ "completions/mean_terminated_length": 81.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2280530333518982,
+ "epoch": 2.3495934959349594,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23682235181331635,
+ "kl": 5.290110129863024e-05,
+ "learning_rate": 1.1185538676315052e-07,
+ "loss": 0.5538168549537659,
+ "num_tokens": 1183973.0,
+ "reward": 0.5785378217697144,
+ "reward_std": 0.29215970635414124,
+ "rewards/true_env_reward_fn/mean": 0.5785378217697144,
+ "rewards/true_env_reward_fn/std": 0.2921597361564636,
+ "step": 289,
+ "step_time": 11.744046860998424
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.264986515045166,
+ "epoch": 2.3577235772357725,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001335782726528123,
+ "kl": 1.7416054106433876e-05,
+ "learning_rate": 1.0924301926274248e-07,
+ "loss": 8.602528396295384e-07,
+ "num_tokens": 1187311.0,
+ "reward": 0.4899469017982483,
+ "reward_std": 0.24049179255962372,
+ "rewards/true_env_reward_fn/mean": 0.4899469017982483,
+ "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "step": 290,
+ "step_time": 4.003069795000556
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.222926139831543,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.209110481198877e-05,
+ "kl": 1.1452370472397888e-05,
+ "learning_rate": 1.0665668633705572e-07,
+ "loss": 5.764911747974111e-07,
+ "num_tokens": 1191370.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 291,
+ "step_time": 4.77749846199913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.313057780265808,
+ "epoch": 2.3739837398373984,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.671779556199908e-05,
+ "kl": 1.1384066056052689e-05,
+ "learning_rate": 1.0409661956807174e-07,
+ "loss": 5.689008162335085e-07,
+ "num_tokens": 1195851.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 292,
+ "step_time": 3.4948791150000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3050158619880676,
+ "epoch": 2.3821138211382116,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15694627165794373,
+ "kl": 2.6891467314271722e-05,
+ "learning_rate": 1.0156304818588308e-07,
+ "loss": -0.044491954147815704,
+ "num_tokens": 1201316.0,
+ "reward": 0.18382371962070465,
+ "reward_std": 0.18414245545864105,
+ "rewards/true_env_reward_fn/mean": 0.18382371962070465,
+ "rewards/true_env_reward_fn/std": 0.18414245545864105,
+ "step": 293,
+ "step_time": 3.900356202000694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2079132199287415,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25304585695266724,
+ "kl": 4.250183792464668e-05,
+ "learning_rate": 9.905619904816749e-08,
+ "loss": 0.008140146732330322,
+ "num_tokens": 1204090.0,
+ "reward": 0.43626630306243896,
+ "reward_std": 0.3027261197566986,
+ "rewards/true_env_reward_fn/mean": 0.43626630306243896,
+ "rewards/true_env_reward_fn/std": 0.3027261197566986,
+ "step": 294,
+ "step_time": 3.1466946830005327
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2324314713478088,
+ "epoch": 2.3983739837398375,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011851672024931759,
+ "kl": 1.4868088328512385e-05,
+ "learning_rate": 9.657629661987531e-08,
+ "loss": 7.534490578109398e-07,
+ "num_tokens": 1206449.0,
+ "reward": 0.7232838273048401,
+ "reward_std": 0.008955853059887886,
+ "rewards/true_env_reward_fn/mean": 0.7232838273048401,
+ "rewards/true_env_reward_fn/std": 0.008955853059887886,
+ "step": 295,
+ "step_time": 3.5492840760016406
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 65.125,
+ "completions/mean_terminated_length": 65.125,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.2110244631767273,
+ "epoch": 2.40650406504065,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18513579666614532,
+ "kl": 6.467000457632821e-05,
+ "learning_rate": 9.412356295313019e-08,
+ "loss": -0.008508354425430298,
+ "num_tokens": 1211294.0,
+ "reward": 0.38126999139785767,
+ "reward_std": 0.22339044511318207,
+ "rewards/true_env_reward_fn/mean": 0.38126999139785767,
+ "rewards/true_env_reward_fn/std": 0.22339043021202087,
+ "step": 296,
+ "step_time": 3.6129159619995335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1828523874282837,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13915051519870758,
+ "kl": 2.76857699645916e-05,
+ "learning_rate": 9.169821766734668e-08,
+ "loss": -0.019797056913375854,
+ "num_tokens": 1214972.0,
+ "reward": 0.5778937339782715,
+ "reward_std": 0.4268997013568878,
+ "rewards/true_env_reward_fn/mean": 0.5778937339782715,
+ "rewards/true_env_reward_fn/std": 0.4268997013568878,
+ "step": 297,
+ "step_time": 3.5299333029997797
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2796242237091064,
+ "epoch": 2.4227642276422765,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23225097358226776,
+ "kl": 5.458398300106637e-05,
+ "learning_rate": 8.930047792956585e-08,
+ "loss": 0.028596192598342896,
+ "num_tokens": 1221117.0,
+ "reward": 0.09695600718259811,
+ "reward_std": 0.23755072057247162,
+ "rewards/true_env_reward_fn/mean": 0.09695600718259811,
+ "rewards/true_env_reward_fn/std": 0.23755072057247162,
+ "step": 298,
+ "step_time": 3.9654863289997593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3131609559059143,
+ "epoch": 2.430894308943089,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15491950511932373,
+ "kl": 2.3678386241954286e-05,
+ "learning_rate": 8.693055843500867e-08,
+ "loss": 0.08264091610908508,
+ "num_tokens": 1226670.0,
+ "reward": 0.22746901214122772,
+ "reward_std": 0.2765822112560272,
+ "rewards/true_env_reward_fn/mean": 0.22746901214122772,
+ "rewards/true_env_reward_fn/std": 0.2765822112560272,
+ "step": 299,
+ "step_time": 3.9300464680000005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 61.75,
+ "completions/mean_terminated_length": 61.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.113481342792511,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14261895418167114,
+ "kl": 2.8488028874562588e-05,
+ "learning_rate": 8.458867138785369e-08,
+ "loss": 0.012870386242866516,
+ "num_tokens": 1230460.0,
+ "reward": 0.6178936958312988,
+ "reward_std": 0.3830615282058716,
+ "rewards/true_env_reward_fn/mean": 0.6178936958312988,
+ "rewards/true_env_reward_fn/std": 0.3830614984035492,
+ "step": 300,
+ "step_time": 4.004705740000645
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 1230460,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-300/training_args.bin b/checkpoint-300/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-300/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-350/README.md b/checkpoint-350/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-350/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-350/adapter_config.json b/checkpoint-350/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-350/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-350/adapter_model.safetensors b/checkpoint-350/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6c75e65da0669d9eb4342a5c1b4417e5b41afa9c
--- /dev/null
+++ b/checkpoint-350/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:533eaa168befd22710329923069f2ecd67a71f79884c32b38445407693b00e42
+size 8731128
diff --git a/checkpoint-350/chat_template.jinja b/checkpoint-350/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-350/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-350/optimizer.pt b/checkpoint-350/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af8766d559bb5cb2ea3b62cd3247d6e28407e714
--- /dev/null
+++ b/checkpoint-350/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a18e246c986121a8a202c83b1f88c188e77479d9bb250438f23fb7a9a8cc5e8
+size 17526842
diff --git a/checkpoint-350/ref/adapter_config.json b/checkpoint-350/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-350/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-350/ref/adapter_model.safetensors b/checkpoint-350/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-350/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-350/rng_state.pth b/checkpoint-350/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4e9803d1d6b0a597a97c9cb707bb958f817e862b
--- /dev/null
+++ b/checkpoint-350/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d57f3e3ca070293c7549a6259323a36e5610ff41b3a9d1573749f4e44986530c
+size 14244
diff --git a/checkpoint-350/scaler.pt b/checkpoint-350/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..770f234ab3c3b67ca35f8fb136afe0a41929398b
--- /dev/null
+++ b/checkpoint-350/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a7ae1d79be755d94030c63fe2e8483abc7a826d35307f4c5e39d3222c511c5c
+size 988
diff --git a/checkpoint-350/scheduler.pt b/checkpoint-350/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..935bf34e5e31b5db56921a8cf308426c15663715
--- /dev/null
+++ b/checkpoint-350/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e41de323bdf0c331883c417e3b3eefaa749e3ae7420b215532e342bfd966244c
+size 1064
diff --git a/checkpoint-350/tokenizer.json b/checkpoint-350/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-350/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-350/tokenizer_config.json b/checkpoint-350/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-350/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-350/trainer_state.json b/checkpoint-350/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..d80a7247e12d58560ecba3c73a502159f5695ce8
--- /dev/null
+++ b/checkpoint-350/trainer_state.json
@@ -0,0 +1,9834 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.845528455284553,
+ "eval_steps": 500,
+ "global_step": 350,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1402934789657593,
+ "epoch": 1.2276422764227641,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010795716661959887,
+ "kl": 1.4652535810455447e-05,
+ "learning_rate": 5.923033215939834e-07,
+ "loss": 7.542968205598299e-07,
+ "num_tokens": 621009.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 151,
+ "step_time": 3.926544339999964
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.190350890159607,
+ "epoch": 1.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19853363931179047,
+ "kl": 2.269768037876929e-05,
+ "learning_rate": 5.889758273918683e-07,
+ "loss": 0.044217392802238464,
+ "num_tokens": 623994.0,
+ "reward": 0.4411996603012085,
+ "reward_std": 0.2517909109592438,
+ "rewards/true_env_reward_fn/mean": 0.4411996603012085,
+ "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "step": 152,
+ "step_time": 3.7339736520007136
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2316884994506836,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.35867181699723e-05,
+ "kl": 1.1579370038816705e-05,
+ "learning_rate": 5.856314121676467e-07,
+ "loss": 5.79387460675207e-07,
+ "num_tokens": 628224.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 153,
+ "step_time": 4.375236807001784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4028943181037903,
+ "epoch": 1.2520325203252032,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25418931245803833,
+ "kl": 3.6890452065563295e-05,
+ "learning_rate": 5.822703753824966e-07,
+ "loss": 0.25599968433380127,
+ "num_tokens": 631183.0,
+ "reward": 0.38683533668518066,
+ "reward_std": 0.43613559007644653,
+ "rewards/true_env_reward_fn/mean": 0.38683533668518066,
+ "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "step": 154,
+ "step_time": 6.055355972999678
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.367663562297821,
+ "epoch": 1.2601626016260163,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331811130046844,
+ "kl": 1.2863993106293492e-05,
+ "learning_rate": 5.788930179859024e-07,
+ "loss": -0.0047044456005096436,
+ "num_tokens": 636230.0,
+ "reward": 0.3489508628845215,
+ "reward_std": 0.13627417385578156,
+ "rewards/true_env_reward_fn/mean": 0.3489508628845215,
+ "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "step": 155,
+ "step_time": 3.9174396130001696
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 68.5,
+ "completions/mean_terminated_length": 68.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2759611010551453,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12738144397735596,
+ "kl": 1.5844128029129934e-05,
+ "learning_rate": 5.754996423887061e-07,
+ "loss": -0.011055335402488708,
+ "num_tokens": 640262.0,
+ "reward": 0.344404935836792,
+ "reward_std": 0.16542991995811462,
+ "rewards/true_env_reward_fn/mean": 0.344404935836792,
+ "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "step": 156,
+ "step_time": 5.706334413998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0904476642608643,
+ "epoch": 1.2764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.733699021628127e-05,
+ "kl": 1.2238857834745431e-05,
+ "learning_rate": 5.720905524360308e-07,
+ "loss": 6.076299996493617e-07,
+ "num_tokens": 645091.0,
+ "reward": 0.4731999933719635,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4731999933719635,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 157,
+ "step_time": 3.7696847109982627
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.0,
+ "completions/mean_terminated_length": 60.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3856677412986755,
+ "epoch": 1.2845528455284554,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18670028448104858,
+ "kl": 1.8415606064081658e-05,
+ "learning_rate": 5.686660533800736e-07,
+ "loss": -0.07078710198402405,
+ "num_tokens": 648179.0,
+ "reward": 0.537517786026001,
+ "reward_std": 0.1451217085123062,
+ "rewards/true_env_reward_fn/mean": 0.537517786026001,
+ "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "step": 158,
+ "step_time": 3.7075291149994882
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 68.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.121916651725769,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11415883898735046,
+ "kl": 1.909901220642496e-05,
+ "learning_rate": 5.652264518527725e-07,
+ "loss": -0.04401372745633125,
+ "num_tokens": 652044.0,
+ "reward": 0.5182899832725525,
+ "reward_std": 0.21869486570358276,
+ "rewards/true_env_reward_fn/mean": 0.5182899832725525,
+ "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "step": 159,
+ "step_time": 3.8929355969994504
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5205118060112,
+ "epoch": 1.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2403375506401062,
+ "kl": 3.7574073758150917e-05,
+ "learning_rate": 5.617720558383508e-07,
+ "loss": 0.26385918259620667,
+ "num_tokens": 656362.0,
+ "reward": 0.41201668977737427,
+ "reward_std": 0.2023741453886032,
+ "rewards/true_env_reward_fn/mean": 0.41201668977737427,
+ "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "step": 160,
+ "step_time": 6.157555950998358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3556928038597107,
+ "epoch": 1.3089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18135924637317657,
+ "kl": 2.6372636057203636e-05,
+ "learning_rate": 5.583031746457407e-07,
+ "loss": -0.10538280755281448,
+ "num_tokens": 659977.0,
+ "reward": 0.4239906072616577,
+ "reward_std": 0.3287450969219208,
+ "rewards/true_env_reward_fn/mean": 0.4239906072616577,
+ "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "step": 161,
+ "step_time": 4.709477423999488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 52.5,
+ "completions/mean_terminated_length": 52.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 0.9743769466876984,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.178797647356987,
+ "kl": 1.2532927030406427e-05,
+ "learning_rate": 5.548201188808869e-07,
+ "loss": -0.04164513945579529,
+ "num_tokens": 661409.0,
+ "reward": 0.8041956424713135,
+ "reward_std": 0.1363772451877594,
+ "rewards/true_env_reward_fn/mean": 0.8041956424713135,
+ "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "step": 162,
+ "step_time": 3.1791253910014348
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.3072250485420227,
+ "epoch": 1.3252032520325203,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16768279671669006,
+ "kl": 1.89386219062726e-05,
+ "learning_rate": 5.513232004189339e-07,
+ "loss": -0.01292814314365387,
+ "num_tokens": 666504.0,
+ "reward": 0.27981066703796387,
+ "reward_std": 0.2949208915233612,
+ "rewards/true_env_reward_fn/mean": 0.27981066703796387,
+ "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "step": 163,
+ "step_time": 3.778431355000066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 51.0,
+ "completions/mean_terminated_length": 51.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1984856128692627,
+ "epoch": 1.3333333333333333,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13123764097690582,
+ "kl": 1.9091786271019373e-05,
+ "learning_rate": 5.478127323763027e-07,
+ "loss": 0.035523779690265656,
+ "num_tokens": 671004.0,
+ "reward": 0.2771500051021576,
+ "reward_std": 0.30146247148513794,
+ "rewards/true_env_reward_fn/mean": 0.2771500051021576,
+ "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "step": 164,
+ "step_time": 3.304021460000513
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 84.875,
+ "completions/mean_terminated_length": 84.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2726752758026123,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15594874322414398,
+ "kl": 2.5981638827943243e-05,
+ "learning_rate": 5.442890290826518e-07,
+ "loss": -0.01398652046918869,
+ "num_tokens": 677307.0,
+ "reward": 0.35573017597198486,
+ "reward_std": 0.25944042205810547,
+ "rewards/true_env_reward_fn/mean": 0.35573017597198486,
+ "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "step": 165,
+ "step_time": 4.972808451999299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 103.625,
+ "completions/mean_terminated_length": 103.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0949090719223022,
+ "epoch": 1.3495934959349594,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07191785424947739,
+ "kl": 1.4828182884230046e-05,
+ "learning_rate": 5.407524060527332e-07,
+ "loss": -0.05063021928071976,
+ "num_tokens": 682076.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 166,
+ "step_time": 8.769379133000257
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1857684254646301,
+ "epoch": 1.3577235772357723,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.923542191041633e-05,
+ "kl": 1.0165251751459436e-05,
+ "learning_rate": 5.37203179958141e-07,
+ "loss": 5.114516170579009e-07,
+ "num_tokens": 685500.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 167,
+ "step_time": 3.1906087530005607
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 74.25,
+ "completions/mean_terminated_length": 74.25,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.527149498462677,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13164250552654266,
+ "kl": 1.8541333702160046e-05,
+ "learning_rate": 5.33641668598956e-07,
+ "loss": -0.2347300797700882,
+ "num_tokens": 688318.0,
+ "reward": 0.7218117713928223,
+ "reward_std": 0.1818692982196808,
+ "rewards/true_env_reward_fn/mean": 0.7218117713928223,
+ "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "step": 168,
+ "step_time": 8.50137474999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 63.0,
+ "completions/min_terminated_length": 63.0,
+ "entropy": 1.3749513030052185,
+ "epoch": 1.3739837398373984,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13246971368789673,
+ "kl": 1.2620409506780561e-05,
+ "learning_rate": 5.300681908752895e-07,
+ "loss": 0.024534843862056732,
+ "num_tokens": 692541.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 169,
+ "step_time": 3.9512340759993094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2032299041748047,
+ "epoch": 1.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11185794323682785,
+ "kl": 1.5517784049734473e-05,
+ "learning_rate": 5.264830667587295e-07,
+ "loss": -0.05245225131511688,
+ "num_tokens": 698064.0,
+ "reward": 0.4444866180419922,
+ "reward_std": 0.32400256395339966,
+ "rewards/true_env_reward_fn/mean": 0.4444866180419922,
+ "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "step": 170,
+ "step_time": 4.471538110999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 95.0,
+ "completions/mean_terminated_length": 95.0,
+ "completions/min_length": 79.0,
+ "completions/min_terminated_length": 79.0,
+ "entropy": 1.4204387068748474,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1646534949541092,
+ "kl": 2.4697198568901513e-05,
+ "learning_rate": 5.228866172636899e-07,
+ "loss": 0.02632315456867218,
+ "num_tokens": 704196.0,
+ "reward": 0.304565966129303,
+ "reward_std": 0.32997164130210876,
+ "rewards/true_env_reward_fn/mean": 0.304565966129303,
+ "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "step": 171,
+ "step_time": 5.0436168590003945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1254178285598755,
+ "epoch": 1.3983739837398375,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15058821439743042,
+ "kl": 1.8407325114822015e-05,
+ "learning_rate": 5.192791644186662e-07,
+ "loss": 0.025478817522525787,
+ "num_tokens": 706411.0,
+ "reward": 0.7279239892959595,
+ "reward_std": 0.11376125365495682,
+ "rewards/true_env_reward_fn/mean": 0.7279239892959595,
+ "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "step": 172,
+ "step_time": 3.2261944119982218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0894773602485657,
+ "epoch": 1.4065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1687694787979126,
+ "kl": 2.6046765015053097e-05,
+ "learning_rate": 5.156610312374013e-07,
+ "loss": -0.056941211223602295,
+ "num_tokens": 711212.0,
+ "reward": 0.4907146692276001,
+ "reward_std": 0.3376546800136566,
+ "rewards/true_env_reward_fn/mean": 0.4907146692276001,
+ "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "step": 173,
+ "step_time": 3.9498180619993946
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 60.125,
+ "completions/mean_terminated_length": 60.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2238691449165344,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19584441184997559,
+ "kl": 4.8285241064149886e-05,
+ "learning_rate": 5.120325416899629e-07,
+ "loss": 0.0766875222325325,
+ "num_tokens": 715409.0,
+ "reward": 0.4593355059623718,
+ "reward_std": 0.3909546136856079,
+ "rewards/true_env_reward_fn/mean": 0.4593355059623718,
+ "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "step": 174,
+ "step_time": 4.100519798999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1763202548027039,
+ "epoch": 1.4227642276422765,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1448555737733841,
+ "kl": 1.2618989785551094e-05,
+ "learning_rate": 5.08394020673734e-07,
+ "loss": -0.012558378279209137,
+ "num_tokens": 722327.0,
+ "reward": 0.15966665744781494,
+ "reward_std": 0.3235519230365753,
+ "rewards/true_env_reward_fn/mean": 0.15966665744781494,
+ "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "step": 175,
+ "step_time": 3.875348296000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2852763533592224,
+ "epoch": 1.4308943089430894,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22303813695907593,
+ "kl": 4.580334098136518e-05,
+ "learning_rate": 5.047457939843227e-07,
+ "loss": -0.09214464575052261,
+ "num_tokens": 726828.0,
+ "reward": 0.25830498337745667,
+ "reward_std": 0.37860655784606934,
+ "rewards/true_env_reward_fn/mean": 0.25830498337745667,
+ "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "step": 176,
+ "step_time": 4.241473076999682
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3609731197357178,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.001985745271668e-05,
+ "kl": 1.4942165307729738e-05,
+ "learning_rate": 5.010881882863893e-07,
+ "loss": 7.44550789022469e-07,
+ "num_tokens": 729930.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 177,
+ "step_time": 3.3902666960002534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2678966522216797,
+ "epoch": 1.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15351757407188416,
+ "kl": 1.6737100395403104e-05,
+ "learning_rate": 4.974215310843967e-07,
+ "loss": 0.041131969541311264,
+ "num_tokens": 732155.0,
+ "reward": 0.7803820371627808,
+ "reward_std": 0.08667682856321335,
+ "rewards/true_env_reward_fn/mean": 0.7803820371627808,
+ "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "step": 178,
+ "step_time": 3.637111981999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.146271526813507,
+ "epoch": 1.4552845528455285,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18466196954250336,
+ "kl": 3.719841197380447e-05,
+ "learning_rate": 4.937461506932859e-07,
+ "loss": 0.029051154851913452,
+ "num_tokens": 735418.0,
+ "reward": 0.40377071499824524,
+ "reward_std": 0.28345924615859985,
+ "rewards/true_env_reward_fn/mean": 0.40377071499824524,
+ "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "step": 179,
+ "step_time": 3.340555791999577
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.3902945518493652,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1020251139998436,
+ "kl": 1.8220500351162627e-05,
+ "learning_rate": 4.900623762090777e-07,
+ "loss": -0.002344265580177307,
+ "num_tokens": 740540.0,
+ "reward": 0.3590222895145416,
+ "reward_std": 0.12487777322530746,
+ "rewards/true_env_reward_fn/mean": 0.3590222895145416,
+ "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "step": 180,
+ "step_time": 7.219923718001155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.517557680606842,
+ "epoch": 1.4715447154471546,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24192294478416443,
+ "kl": 4.1268089262302965e-05,
+ "learning_rate": 4.863705374794055e-07,
+ "loss": 0.09132950007915497,
+ "num_tokens": 744723.0,
+ "reward": 0.23991000652313232,
+ "reward_std": 0.2837013602256775,
+ "rewards/true_env_reward_fn/mean": 0.23991000652313232,
+ "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "step": 181,
+ "step_time": 3.696339096999509
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.249614655971527,
+ "epoch": 1.4796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1566745787858963,
+ "kl": 2.6629099920683075e-05,
+ "learning_rate": 4.826709650739811e-07,
+ "loss": 0.003972277045249939,
+ "num_tokens": 748979.0,
+ "reward": 0.4935140311717987,
+ "reward_std": 0.41978561878204346,
+ "rewards/true_env_reward_fn/mean": 0.4935140311717987,
+ "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "step": 182,
+ "step_time": 3.316512920000605
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1666916608810425,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001005820304271765,
+ "kl": 1.2246940059412736e-05,
+ "learning_rate": 4.789639902549948e-07,
+ "loss": 6.330609494398232e-07,
+ "num_tokens": 751323.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 183,
+ "step_time": 3.7099916660008603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3246222138404846,
+ "epoch": 1.4959349593495934,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18140996992588043,
+ "kl": 3.1042441150930244e-05,
+ "learning_rate": 4.752499449474535e-07,
+ "loss": -0.022353097796440125,
+ "num_tokens": 755494.0,
+ "reward": 0.4561777412891388,
+ "reward_std": 0.2439236342906952,
+ "rewards/true_env_reward_fn/mean": 0.4561777412891388,
+ "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "step": 184,
+ "step_time": 3.7916486710000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 75.75,
+ "completions/mean_terminated_length": 75.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.2625707983970642,
+ "epoch": 1.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11702944338321686,
+ "kl": 1.503958355897339e-05,
+ "learning_rate": 4.715291617094607e-07,
+ "loss": 0.023916304111480713,
+ "num_tokens": 758432.0,
+ "reward": 0.5015827417373657,
+ "reward_std": 0.1783808320760727,
+ "rewards/true_env_reward_fn/mean": 0.5015827417373657,
+ "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "step": 185,
+ "step_time": 4.226409274000616
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1460023522377014,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11851620674133301,
+ "kl": 1.3728345948038623e-05,
+ "learning_rate": 4.678019737024387e-07,
+ "loss": 0.0831337422132492,
+ "num_tokens": 764336.0,
+ "reward": 0.19094166159629822,
+ "reward_std": 0.30934420228004456,
+ "rewards/true_env_reward_fn/mean": 0.19094166159629822,
+ "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "step": 186,
+ "step_time": 3.563357556000483
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1291148662567139,
+ "epoch": 1.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.864256960805506e-05,
+ "kl": 1.081683785741916e-05,
+ "learning_rate": 4.6406871466129704e-07,
+ "loss": 5.412177870312007e-07,
+ "num_tokens": 766608.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 187,
+ "step_time": 3.534869859002356
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 92.0,
+ "completions/mean_terminated_length": 92.0,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3645328283309937,
+ "epoch": 1.5284552845528454,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18168245255947113,
+ "kl": 2.6857565899263136e-05,
+ "learning_rate": 4.6032971886454956e-07,
+ "loss": -0.005156125873327255,
+ "num_tokens": 774496.0,
+ "reward": 0.11249999701976776,
+ "reward_std": 0.20856082439422607,
+ "rewards/true_env_reward_fn/mean": 0.11249999701976776,
+ "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "step": 188,
+ "step_time": 8.34005261099992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1495982110500336,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14486820995807648,
+ "kl": 1.3202762147557223e-05,
+ "learning_rate": 4.5658532110438337e-07,
+ "loss": -0.0010610297322273254,
+ "num_tokens": 777186.0,
+ "reward": 0.5879127383232117,
+ "reward_std": 0.05142820253968239,
+ "rewards/true_env_reward_fn/mean": 0.5879127383232117,
+ "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "step": 189,
+ "step_time": 3.4856022139993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.241140365600586,
+ "epoch": 1.5447154471544715,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001250067143701017,
+ "kl": 1.5482702110602986e-05,
+ "learning_rate": 4.52835856656681e-07,
+ "loss": 7.80837922320643e-07,
+ "num_tokens": 779965.0,
+ "reward": 0.6861198544502258,
+ "reward_std": 0.08807206153869629,
+ "rewards/true_env_reward_fn/mean": 0.6861198544502258,
+ "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "step": 190,
+ "step_time": 3.904181735999373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2944807410240173,
+ "epoch": 1.5528455284552845,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2030111849308014,
+ "kl": 3.585523518268019e-05,
+ "learning_rate": 4.490816612509991e-07,
+ "loss": 0.0143373291939497,
+ "num_tokens": 786140.0,
+ "reward": 0.35173332691192627,
+ "reward_std": 0.18115806579589844,
+ "rewards/true_env_reward_fn/mean": 0.35173332691192627,
+ "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "step": 191,
+ "step_time": 4.005758510000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 58.0,
+ "completions/mean_terminated_length": 58.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2177271246910095,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11188288033008575,
+ "kl": 2.3622495064046234e-05,
+ "learning_rate": 4.45323071040508e-07,
+ "loss": -0.02369789034128189,
+ "num_tokens": 790424.0,
+ "reward": 0.4888629913330078,
+ "reward_std": 0.23310808837413788,
+ "rewards/true_env_reward_fn/mean": 0.4888629913330078,
+ "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "step": 192,
+ "step_time": 3.2518814809991454
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.192966103553772,
+ "epoch": 1.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1378823071718216,
+ "kl": 1.7358055174554465e-05,
+ "learning_rate": 4.4156042257189143e-07,
+ "loss": 0.06256310641765594,
+ "num_tokens": 794521.0,
+ "reward": 0.5219699740409851,
+ "reward_std": 0.06214587390422821,
+ "rewards/true_env_reward_fn/mean": 0.5219699740409851,
+ "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "step": 193,
+ "step_time": 4.222739491999164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1770159006118774,
+ "epoch": 1.5772357723577235,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19550059735774994,
+ "kl": 2.4871268578863237e-05,
+ "learning_rate": 4.377940527552125e-07,
+ "loss": 0.05841376632452011,
+ "num_tokens": 798194.0,
+ "reward": 0.42302167415618896,
+ "reward_std": 0.2911272346973419,
+ "rewards/true_env_reward_fn/mean": 0.42302167415618896,
+ "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "step": 194,
+ "step_time": 3.8170270639984665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.5244255661964417,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1872202306985855,
+ "kl": 2.5990483663917985e-05,
+ "learning_rate": 4.340242988337462e-07,
+ "loss": -0.044112429022789,
+ "num_tokens": 802802.0,
+ "reward": 0.33745431900024414,
+ "reward_std": 0.22955451905727386,
+ "rewards/true_env_reward_fn/mean": 0.33745431900024414,
+ "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "step": 195,
+ "step_time": 4.27381555500142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.345891296863556,
+ "epoch": 1.5934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16891054809093475,
+ "kl": 2.7261638024356216e-05,
+ "learning_rate": 4.3025149835378275e-07,
+ "loss": -0.139386385679245,
+ "num_tokens": 807881.0,
+ "reward": 0.3240283131599426,
+ "reward_std": 0.2803676128387451,
+ "rewards/true_env_reward_fn/mean": 0.3240283131599426,
+ "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "step": 196,
+ "step_time": 4.981287381999209
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.246802031993866,
+ "epoch": 1.6016260162601625,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2096729576587677,
+ "kl": 3.5958016269432846e-05,
+ "learning_rate": 4.2647598913440264e-07,
+ "loss": -0.02941281348466873,
+ "num_tokens": 812500.0,
+ "reward": 0.4126526415348053,
+ "reward_std": 0.36393746733665466,
+ "rewards/true_env_reward_fn/mean": 0.4126526415348053,
+ "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "step": 197,
+ "step_time": 3.101726017999681
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4369062185287476,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1862105429172516,
+ "kl": 4.7646244638599455e-05,
+ "learning_rate": 4.2269810923722965e-07,
+ "loss": 0.0521145761013031,
+ "num_tokens": 818718.0,
+ "reward": 0.26869943737983704,
+ "reward_std": 0.1483483463525772,
+ "rewards/true_env_reward_fn/mean": 0.26869943737983704,
+ "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "step": 198,
+ "step_time": 4.287780451000799
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.192937195301056,
+ "epoch": 1.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2013663798570633,
+ "kl": 1.8760739294521045e-05,
+ "learning_rate": 4.189181969361588e-07,
+ "loss": 0.07236722111701965,
+ "num_tokens": 825728.0,
+ "reward": 0.23110000789165497,
+ "reward_std": 0.23212090134620667,
+ "rewards/true_env_reward_fn/mean": 0.23110000789165497,
+ "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "step": 199,
+ "step_time": 4.708717262998107
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.3792839050292969,
+ "epoch": 1.6260162601626016,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.7985117463395e-05,
+ "kl": 1.4280476534622721e-05,
+ "learning_rate": 4.1513659068706814e-07,
+ "loss": 7.153485626076872e-07,
+ "num_tokens": 830318.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 200,
+ "step_time": 3.9055351140013954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0505937337875366,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13188514113426208,
+ "kl": 2.842090088961413e-05,
+ "learning_rate": 4.1135362909751326e-07,
+ "loss": -0.0017508119344711304,
+ "num_tokens": 834933.0,
+ "reward": 0.36545002460479736,
+ "reward_std": 0.24526984989643097,
+ "rewards/true_env_reward_fn/mean": 0.36545002460479736,
+ "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "step": 201,
+ "step_time": 3.895525625997834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.3750707507133484,
+ "epoch": 1.6422764227642277,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1798955649137497,
+ "kl": 2.5428611479583196e-05,
+ "learning_rate": 4.075696508964076e-07,
+ "loss": 0.1843666434288025,
+ "num_tokens": 838076.0,
+ "reward": 0.44641831517219543,
+ "reward_std": 0.30040720105171204,
+ "rewards/true_env_reward_fn/mean": 0.44641831517219543,
+ "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "step": 202,
+ "step_time": 6.74463491500137
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 79.125,
+ "completions/mean_terminated_length": 79.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3369249105453491,
+ "epoch": 1.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11249764263629913,
+ "kl": 1.1453126717242412e-05,
+ "learning_rate": 4.0378499490369267e-07,
+ "loss": -0.08210685849189758,
+ "num_tokens": 841933.0,
+ "reward": 0.3737962245941162,
+ "reward_std": 0.13184049725532532,
+ "rewards/true_env_reward_fn/mean": 0.3737962245941162,
+ "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "step": 203,
+ "step_time": 5.5639925510004105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.5737199783325195,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19394491612911224,
+ "kl": 5.05317857459886e-05,
+ "learning_rate": 4e-07,
+ "loss": 0.05622926354408264,
+ "num_tokens": 847711.0,
+ "reward": 0.10725000500679016,
+ "reward_std": 0.19980257749557495,
+ "rewards/true_env_reward_fn/mean": 0.10725000500679016,
+ "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "step": 204,
+ "step_time": 4.918089437000162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.119917094707489,
+ "epoch": 1.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10553985089063644,
+ "kl": 1.2793303994840244e-05,
+ "learning_rate": 3.9621500509630725e-07,
+ "loss": -0.011355768889188766,
+ "num_tokens": 849519.0,
+ "reward": 0.6593211889266968,
+ "reward_std": 0.11862105131149292,
+ "rewards/true_env_reward_fn/mean": 0.6593211889266968,
+ "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "step": 205,
+ "step_time": 4.127652793999005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 75.125,
+ "completions/mean_terminated_length": 75.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3856809735298157,
+ "epoch": 1.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1911258101463318,
+ "kl": 4.2569914512569085e-05,
+ "learning_rate": 3.9243034910359247e-07,
+ "loss": 0.10561336576938629,
+ "num_tokens": 854156.0,
+ "reward": 0.2516202926635742,
+ "reward_std": 0.22357939183712006,
+ "rewards/true_env_reward_fn/mean": 0.2516202926635742,
+ "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "step": 206,
+ "step_time": 4.714620994000143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.4012945890426636,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226125180721283,
+ "kl": 5.4918069963605376e-05,
+ "learning_rate": 3.886463709024868e-07,
+ "loss": 0.012949362397193909,
+ "num_tokens": 856151.0,
+ "reward": 0.6204804182052612,
+ "reward_std": 0.44673967361450195,
+ "rewards/true_env_reward_fn/mean": 0.6204804182052612,
+ "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "step": 207,
+ "step_time": 4.094810713999323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3708943128585815,
+ "epoch": 1.6910569105691056,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12903675436973572,
+ "kl": 1.4145812656352064e-05,
+ "learning_rate": 3.8486340931293187e-07,
+ "loss": -0.07838249206542969,
+ "num_tokens": 859380.0,
+ "reward": 0.6400156021118164,
+ "reward_std": 0.1022576317191124,
+ "rewards/true_env_reward_fn/mean": 0.6400156021118164,
+ "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "step": 208,
+ "step_time": 4.297900428997309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.75,
+ "completions/mean_terminated_length": 56.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2782961130142212,
+ "epoch": 1.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12437833100557327,
+ "kl": 1.4473939245362999e-05,
+ "learning_rate": 3.8108180306384135e-07,
+ "loss": -0.036324724555015564,
+ "num_tokens": 862714.0,
+ "reward": 0.43419933319091797,
+ "reward_std": 0.15345513820648193,
+ "rewards/true_env_reward_fn/mean": 0.43419933319091797,
+ "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "step": 209,
+ "step_time": 3.47861851900052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 123.875,
+ "completions/mean_terminated_length": 68.42857360839844,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.5170292258262634,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1543467938899994,
+ "kl": 2.3121557205740828e-05,
+ "learning_rate": 3.7730189076277037e-07,
+ "loss": -0.3810324966907501,
+ "num_tokens": 869789.0,
+ "reward": 0.3795333504676819,
+ "reward_std": 0.13815106451511383,
+ "rewards/true_env_reward_fn/mean": 0.3795333504676819,
+ "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "step": 210,
+ "step_time": 20.698896928999602
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2324861884117126,
+ "epoch": 1.7154471544715446,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11762262135744095,
+ "kl": 1.9743249595194357e-05,
+ "learning_rate": 3.735240108655973e-07,
+ "loss": -0.09104303270578384,
+ "num_tokens": 872604.0,
+ "reward": 0.5916227102279663,
+ "reward_std": 0.1752101182937622,
+ "rewards/true_env_reward_fn/mean": 0.5916227102279663,
+ "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "step": 211,
+ "step_time": 5.218213289999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.4448966979980469,
+ "epoch": 1.7235772357723578,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1924143135547638,
+ "kl": 3.186432604707079e-05,
+ "learning_rate": 3.697485016462174e-07,
+ "loss": 0.025449808686971664,
+ "num_tokens": 877921.0,
+ "reward": 0.1028124988079071,
+ "reward_std": 0.2254277467727661,
+ "rewards/true_env_reward_fn/mean": 0.1028124988079071,
+ "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "step": 212,
+ "step_time": 3.415528882000217
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.185910701751709,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001214127623825334,
+ "kl": 1.3199866316426778e-05,
+ "learning_rate": 3.659757011662538e-07,
+ "loss": 6.776077725589857e-07,
+ "num_tokens": 880344.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 213,
+ "step_time": 3.5156538789997285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 90.75,
+ "completions/mean_terminated_length": 90.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2697569727897644,
+ "epoch": 1.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1553589552640915,
+ "kl": 1.621047795197228e-05,
+ "learning_rate": 3.622059472447875e-07,
+ "loss": -0.111361563205719,
+ "num_tokens": 885006.0,
+ "reward": 0.5490846633911133,
+ "reward_std": 0.14710450172424316,
+ "rewards/true_env_reward_fn/mean": 0.5490846633911133,
+ "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "step": 214,
+ "step_time": 7.051423932000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1904898285865784,
+ "epoch": 1.7479674796747968,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19009968638420105,
+ "kl": 1.6167180092452327e-05,
+ "learning_rate": 3.5843957742810864e-07,
+ "loss": -0.006048411130905151,
+ "num_tokens": 888255.0,
+ "reward": 0.5893601179122925,
+ "reward_std": 0.16021940112113953,
+ "rewards/true_env_reward_fn/mean": 0.5893601179122925,
+ "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "step": 215,
+ "step_time": 3.0270869319992926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2750649452209473,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1521027386188507,
+ "kl": 4.317680577514693e-05,
+ "learning_rate": 3.5467692895949205e-07,
+ "loss": -0.04247616231441498,
+ "num_tokens": 892490.0,
+ "reward": 0.1841849982738495,
+ "reward_std": 0.30015870928764343,
+ "rewards/true_env_reward_fn/mean": 0.1841849982738495,
+ "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "step": 216,
+ "step_time": 3.461749838998003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2730335593223572,
+ "epoch": 1.7642276422764227,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11831019073724747,
+ "kl": 2.1095927877468057e-05,
+ "learning_rate": 3.509183387490009e-07,
+ "loss": -0.051231447607278824,
+ "num_tokens": 894688.0,
+ "reward": 0.49520131945610046,
+ "reward_std": 0.3804744482040405,
+ "rewards/true_env_reward_fn/mean": 0.49520131945610046,
+ "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "step": 217,
+ "step_time": 3.7184635590019752
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 47.75,
+ "completions/mean_terminated_length": 47.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3015827536582947,
+ "epoch": 1.7723577235772359,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21724702417850494,
+ "kl": 3.562447545846226e-05,
+ "learning_rate": 3.471641433433191e-07,
+ "loss": -0.028775859624147415,
+ "num_tokens": 899066.0,
+ "reward": 0.3376166820526123,
+ "reward_std": 0.21976198256015778,
+ "rewards/true_env_reward_fn/mean": 0.3376166820526123,
+ "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "step": 218,
+ "step_time": 3.344433074000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1390373706817627,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001077545020962134,
+ "kl": 1.554161144667887e-05,
+ "learning_rate": 3.434146788956166e-07,
+ "loss": 7.676237032683275e-07,
+ "num_tokens": 902263.0,
+ "reward": 0.5682899951934814,
+ "reward_std": 0.1217179074883461,
+ "rewards/true_env_reward_fn/mean": 0.5682899951934814,
+ "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "step": 219,
+ "step_time": 3.0424018219982827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1991714239120483,
+ "epoch": 1.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11428863555192947,
+ "kl": 1.1265870853094384e-05,
+ "learning_rate": 3.3967028113545045e-07,
+ "loss": -0.02927359938621521,
+ "num_tokens": 906206.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 220,
+ "step_time": 3.5531271640011255
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2546668648719788,
+ "epoch": 1.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1504630595445633,
+ "kl": 1.5663241811125772e-05,
+ "learning_rate": 3.3593128533870314e-07,
+ "loss": 0.04806854575872421,
+ "num_tokens": 909149.0,
+ "reward": 0.49599751830101013,
+ "reward_std": 0.15429075062274933,
+ "rewards/true_env_reward_fn/mean": 0.49599751830101013,
+ "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "step": 221,
+ "step_time": 3.1345955030010373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1428714394569397,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.310600969707593e-05,
+ "kl": 1.1706195891747484e-05,
+ "learning_rate": 3.321980262975613e-07,
+ "loss": 5.957842290627013e-07,
+ "num_tokens": 914211.0,
+ "reward": 0.3258306384086609,
+ "reward_std": 0.4338511824607849,
+ "rewards/true_env_reward_fn/mean": 0.3258306384086609,
+ "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "step": 222,
+ "step_time": 3.8445859539988305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1629019975662231,
+ "epoch": 1.8130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010947151895379648,
+ "kl": 1.3530024261854123e-05,
+ "learning_rate": 3.2847083829053923e-07,
+ "loss": 6.723923888785066e-07,
+ "num_tokens": 916850.0,
+ "reward": 0.5956059694290161,
+ "reward_std": 0.09251586347818375,
+ "rewards/true_env_reward_fn/mean": 0.5956059694290161,
+ "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "step": 223,
+ "step_time": 3.3021794950000185
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1237311363220215,
+ "epoch": 1.821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14958374202251434,
+ "kl": 4.083753810846247e-05,
+ "learning_rate": 3.2475005505254657e-07,
+ "loss": 0.09789139032363892,
+ "num_tokens": 919901.0,
+ "reward": 0.4179220199584961,
+ "reward_std": 0.2486819326877594,
+ "rewards/true_env_reward_fn/mean": 0.4179220199584961,
+ "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "step": 224,
+ "step_time": 4.954825423001239
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 52.375,
+ "completions/mean_terminated_length": 52.375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1394256949424744,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1480269879102707,
+ "kl": 3.2915593692450784e-05,
+ "learning_rate": 3.210360097450052e-07,
+ "loss": 0.11319300532341003,
+ "num_tokens": 923840.0,
+ "reward": 0.4335233271121979,
+ "reward_std": 0.31114333868026733,
+ "rewards/true_env_reward_fn/mean": 0.4335233271121979,
+ "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "step": 225,
+ "step_time": 3.777259659000265
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.354669451713562,
+ "epoch": 1.8373983739837398,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13672730326652527,
+ "kl": 2.2323275516100693e-05,
+ "learning_rate": 3.173290349260188e-07,
+ "loss": -0.0521523654460907,
+ "num_tokens": 927882.0,
+ "reward": 0.57341468334198,
+ "reward_std": 0.11576741933822632,
+ "rewards/true_env_reward_fn/mean": 0.57341468334198,
+ "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "step": 226,
+ "step_time": 3.613498073998926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1750767230987549,
+ "epoch": 1.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.230674147605896,
+ "kl": 1.800864629331045e-05,
+ "learning_rate": 3.136294625205945e-07,
+ "loss": 0.058730173856019974,
+ "num_tokens": 930225.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 227,
+ "step_time": 3.3819083769976714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 40.875,
+ "completions/mean_terminated_length": 40.875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2825847864151,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.30391862988471985,
+ "kl": 2.0106223928451072e-05,
+ "learning_rate": 3.0993762379092235e-07,
+ "loss": -0.10232458263635635,
+ "num_tokens": 933048.0,
+ "reward": 0.45667415857315063,
+ "reward_std": 0.26290765404701233,
+ "rewards/true_env_reward_fn/mean": 0.45667415857315063,
+ "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "step": 228,
+ "step_time": 2.698590726000475
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 41.125,
+ "completions/mean_terminated_length": 41.125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3104369640350342,
+ "epoch": 1.8617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.28224605321884155,
+ "kl": 6.227439189387951e-05,
+ "learning_rate": 3.06253849306714e-07,
+ "loss": -0.008679002523422241,
+ "num_tokens": 938217.0,
+ "reward": 0.2567799985408783,
+ "reward_std": 0.2555168867111206,
+ "rewards/true_env_reward_fn/mean": 0.2567799985408783,
+ "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "step": 229,
+ "step_time": 2.7451177700022527
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0164751410484314,
+ "epoch": 1.8699186991869918,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011273886048002169,
+ "kl": 1.1985231139988173e-05,
+ "learning_rate": 3.0257846891560323e-07,
+ "loss": 5.987301960885816e-07,
+ "num_tokens": 941049.0,
+ "reward": 0.6971603631973267,
+ "reward_std": 0.2594861686229706,
+ "rewards/true_env_reward_fn/mean": 0.6971603631973267,
+ "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "step": 230,
+ "step_time": 2.917641182999432
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.1991845965385437,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015363915008492768,
+ "kl": 1.4284144981502322e-05,
+ "learning_rate": 2.989118117136107e-07,
+ "loss": 6.798551339670666e-07,
+ "num_tokens": 945403.0,
+ "reward": 0.31745320558547974,
+ "reward_std": 0.42489534616470337,
+ "rewards/true_env_reward_fn/mean": 0.31745320558547974,
+ "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "step": 231,
+ "step_time": 5.5585464220002905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2020843029022217,
+ "epoch": 1.886178861788618,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13684362173080444,
+ "kl": 3.651866154541494e-05,
+ "learning_rate": 2.952542060156773e-07,
+ "loss": 0.025869816541671753,
+ "num_tokens": 949126.0,
+ "reward": 0.3984018564224243,
+ "reward_std": 0.4202974736690521,
+ "rewards/true_env_reward_fn/mean": 0.3984018564224243,
+ "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "step": 232,
+ "step_time": 3.4383463869999105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 58.625,
+ "completions/mean_terminated_length": 58.625,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2224581837654114,
+ "epoch": 1.8943089430894309,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20249564945697784,
+ "kl": 3.106597978330683e-05,
+ "learning_rate": 2.9160597932626605e-07,
+ "loss": -0.003915777429938316,
+ "num_tokens": 952815.0,
+ "reward": 0.4141089916229248,
+ "reward_std": 0.3064958453178406,
+ "rewards/true_env_reward_fn/mean": 0.4141089916229248,
+ "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "step": 233,
+ "step_time": 3.6410487339999236
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1212781071662903,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14502985775470734,
+ "kl": 3.0683338081871625e-05,
+ "learning_rate": 2.879674583100372e-07,
+ "loss": -0.0860406681895256,
+ "num_tokens": 958656.0,
+ "reward": 0.30326664447784424,
+ "reward_std": 0.24800051748752594,
+ "rewards/true_env_reward_fn/mean": 0.30326664447784424,
+ "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "step": 234,
+ "step_time": 3.8798253620007017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 60.75,
+ "completions/mean_terminated_length": 60.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.425286054611206,
+ "epoch": 1.910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2035023272037506,
+ "kl": 5.512987627298571e-05,
+ "learning_rate": 2.843389687625986e-07,
+ "loss": -0.010439477860927582,
+ "num_tokens": 962638.0,
+ "reward": 0.3157375156879425,
+ "reward_std": 0.46047845482826233,
+ "rewards/true_env_reward_fn/mean": 0.3157375156879425,
+ "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "step": 235,
+ "step_time": 4.114513064998391
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 77.625,
+ "completions/mean_terminated_length": 77.625,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4385854601860046,
+ "epoch": 1.91869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12982772290706635,
+ "kl": 1.2591926861205138e-05,
+ "learning_rate": 2.807208355813339e-07,
+ "loss": 0.09093751758337021,
+ "num_tokens": 965755.0,
+ "reward": 0.6140732765197754,
+ "reward_std": 0.27462607622146606,
+ "rewards/true_env_reward_fn/mean": 0.6140732765197754,
+ "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "step": 236,
+ "step_time": 5.147667763001664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.3476852178573608,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14480431377887726,
+ "kl": 3.8014684832887724e-05,
+ "learning_rate": 2.771133827363101e-07,
+ "loss": -0.07322391867637634,
+ "num_tokens": 970918.0,
+ "reward": 0.3427826166152954,
+ "reward_std": 0.42430612444877625,
+ "rewards/true_env_reward_fn/mean": 0.3427826166152954,
+ "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "step": 237,
+ "step_time": 4.67846887900123
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 221.0,
+ "completions/max_terminated_length": 221.0,
+ "completions/mean_length": 94.25,
+ "completions/mean_terminated_length": 94.25,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3279914855957031,
+ "epoch": 1.934959349593496,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10826019197702408,
+ "kl": 1.3074863090878353e-05,
+ "learning_rate": 2.7351693324127037e-07,
+ "loss": -0.05556309223175049,
+ "num_tokens": 975120.0,
+ "reward": 0.64573073387146,
+ "reward_std": 0.22739914059638977,
+ "rewards/true_env_reward_fn/mean": 0.64573073387146,
+ "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "step": 238,
+ "step_time": 9.220254810001279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3695034384727478,
+ "epoch": 1.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12166028469800949,
+ "kl": 2.6563114261080045e-05,
+ "learning_rate": 2.6993180912471055e-07,
+ "loss": -0.05334407091140747,
+ "num_tokens": 980254.0,
+ "reward": 0.37203267216682434,
+ "reward_std": 0.20089927315711975,
+ "rewards/true_env_reward_fn/mean": 0.37203267216682434,
+ "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "step": 239,
+ "step_time": 4.224964968001586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.75,
+ "completions/mean_terminated_length": 53.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.247464120388031,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2346416860818863,
+ "kl": 5.1520751185307745e-05,
+ "learning_rate": 2.6635833140104405e-07,
+ "loss": -0.0905834436416626,
+ "num_tokens": 983244.0,
+ "reward": 0.575507640838623,
+ "reward_std": 0.17391785979270935,
+ "rewards/true_env_reward_fn/mean": 0.575507640838623,
+ "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "step": 240,
+ "step_time": 3.357481237999309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2175387144088745,
+ "epoch": 1.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1870066374540329,
+ "kl": 2.2185965462995227e-05,
+ "learning_rate": 2.6279682004185894e-07,
+ "loss": -0.07365687191486359,
+ "num_tokens": 985574.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 241,
+ "step_time": 4.527591582998866
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2727615237236023,
+ "epoch": 1.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.226071804529056e-05,
+ "kl": 1.4014385214977665e-05,
+ "learning_rate": 2.592475939472668e-07,
+ "loss": 7.015369192231447e-07,
+ "num_tokens": 988868.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 242,
+ "step_time": 4.238274277000528
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1519948840141296,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12447996437549591,
+ "kl": 4.700180943473242e-05,
+ "learning_rate": 2.557109709173482e-07,
+ "loss": 0.08819369971752167,
+ "num_tokens": 994068.0,
+ "reward": 0.3696666657924652,
+ "reward_std": 0.18488828837871552,
+ "rewards/true_env_reward_fn/mean": 0.3696666657924652,
+ "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "step": 243,
+ "step_time": 3.3367313500002638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.25,
+ "completions/mean_terminated_length": 68.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0537148416042328,
+ "epoch": 1.9837398373983741,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17681685090065002,
+ "kl": 3.6240851841284893e-05,
+ "learning_rate": 2.521872676236972e-07,
+ "loss": 0.05281040072441101,
+ "num_tokens": 1000650.0,
+ "reward": 0.12209999561309814,
+ "reward_std": 0.2502918243408203,
+ "rewards/true_env_reward_fn/mean": 0.12209999561309814,
+ "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "step": 244,
+ "step_time": 5.2957401019993995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 61.0,
+ "completions/min_terminated_length": 61.0,
+ "entropy": 1.3126497864723206,
+ "epoch": 1.9918699186991868,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16036106646060944,
+ "kl": 6.061139720259234e-05,
+ "learning_rate": 2.48676799581066e-07,
+ "loss": 0.09418506920337677,
+ "num_tokens": 1005737.0,
+ "reward": 0.31175702810287476,
+ "reward_std": 0.38867074251174927,
+ "rewards/true_env_reward_fn/mean": 0.31175702810287476,
+ "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "step": 245,
+ "step_time": 6.259088058999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 48.5,
+ "completions/mean_terminated_length": 48.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.21333646774292,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23997871577739716,
+ "kl": 3.1378609492094256e-05,
+ "learning_rate": 2.4517988111911313e-07,
+ "loss": 0.010592922568321228,
+ "num_tokens": 1010869.0,
+ "reward": 0.33381664752960205,
+ "reward_std": 0.18213039636611938,
+ "rewards/true_env_reward_fn/mean": 0.33381664752960205,
+ "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "step": 246,
+ "step_time": 3.111915630997828
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.3962982892990112,
+ "epoch": 2.008130081300813,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21355785429477692,
+ "kl": 3.789625407080166e-05,
+ "learning_rate": 2.4169682535425927e-07,
+ "loss": 0.025682777166366577,
+ "num_tokens": 1014876.0,
+ "reward": 0.35749268531799316,
+ "reward_std": 0.29738906025886536,
+ "rewards/true_env_reward_fn/mean": 0.35749268531799316,
+ "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "step": 247,
+ "step_time": 3.35338095500083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 73.875,
+ "completions/mean_terminated_length": 73.875,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.463137686252594,
+ "epoch": 2.016260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18188992142677307,
+ "kl": 1.746804719005013e-05,
+ "learning_rate": 2.382279441616492e-07,
+ "loss": -0.17857304215431213,
+ "num_tokens": 1018383.0,
+ "reward": 0.5329012274742126,
+ "reward_std": 0.055823445320129395,
+ "rewards/true_env_reward_fn/mean": 0.5329012274742126,
+ "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "step": 248,
+ "step_time": 5.210386754000865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 79.25,
+ "completions/mean_terminated_length": 79.25,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.4478936195373535,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.744662434561178e-05,
+ "kl": 1.3336490155779757e-05,
+ "learning_rate": 2.3477354814722762e-07,
+ "loss": 6.725406365148956e-07,
+ "num_tokens": 1022753.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 249,
+ "step_time": 8.798317029002646
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.37166029214859,
+ "epoch": 2.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1542455554008484,
+ "kl": 2.0379054603836266e-05,
+ "learning_rate": 2.313339466199264e-07,
+ "loss": -0.037539318203926086,
+ "num_tokens": 1025971.0,
+ "reward": 0.6065863966941833,
+ "reward_std": 0.032470256090164185,
+ "rewards/true_env_reward_fn/mean": 0.6065863966941833,
+ "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "step": 250,
+ "step_time": 4.096263454999644
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4042693972587585,
+ "epoch": 2.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.163822203874588,
+ "kl": 3.525477495713858e-05,
+ "learning_rate": 2.2790944756396916e-07,
+ "loss": 0.03408379852771759,
+ "num_tokens": 1029415.0,
+ "reward": 0.37829869985580444,
+ "reward_std": 0.2773255407810211,
+ "rewards/true_env_reward_fn/mean": 0.37829869985580444,
+ "rewards/true_env_reward_fn/std": 0.2773255407810211,
+ "step": 251,
+ "step_time": 3.9464334140011488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2268111109733582,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1961166262626648,
+ "kl": 3.528672823449597e-05,
+ "learning_rate": 2.2450035761129391e-07,
+ "loss": 0.2999379336833954,
+ "num_tokens": 1032213.0,
+ "reward": 0.6772161722183228,
+ "reward_std": 0.31218820810317993,
+ "rewards/true_env_reward_fn/mean": 0.6772161722183228,
+ "rewards/true_env_reward_fn/std": 0.31218820810317993,
+ "step": 252,
+ "step_time": 7.437029113001699
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.360656499862671,
+ "epoch": 2.0569105691056913,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24334783852100372,
+ "kl": 5.1042834456893615e-05,
+ "learning_rate": 2.2110698201409787e-07,
+ "loss": 0.05509951710700989,
+ "num_tokens": 1038598.0,
+ "reward": 0.2947666645050049,
+ "reward_std": 0.19891902804374695,
+ "rewards/true_env_reward_fn/mean": 0.2947666645050049,
+ "rewards/true_env_reward_fn/std": 0.19891902804374695,
+ "step": 253,
+ "step_time": 3.7359043900014512
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 74.75,
+ "completions/mean_terminated_length": 74.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4314632415771484,
+ "epoch": 2.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1960419863462448,
+ "kl": 6.17889963905327e-05,
+ "learning_rate": 2.1772962461750342e-07,
+ "loss": 0.0954262986779213,
+ "num_tokens": 1043268.0,
+ "reward": 0.4036714732646942,
+ "reward_std": 0.42137831449508667,
+ "rewards/true_env_reward_fn/mean": 0.4036714732646942,
+ "rewards/true_env_reward_fn/std": 0.42137834429740906,
+ "step": 254,
+ "step_time": 5.565175547999388
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0521443486213684,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.464648271910846e-05,
+ "kl": 1.2018902907584561e-05,
+ "learning_rate": 2.1436858783235338e-07,
+ "loss": 6.008343689245521e-07,
+ "num_tokens": 1046517.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 255,
+ "step_time": 4.379171047001364
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.091518223285675,
+ "epoch": 2.08130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.263061656383798e-05,
+ "kl": 1.4573892713087844e-05,
+ "learning_rate": 2.110241726081317e-07,
+ "loss": 7.304333848878741e-07,
+ "num_tokens": 1050439.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 256,
+ "step_time": 3.8379976090000127
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2301559448242188,
+ "epoch": 2.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17180238664150238,
+ "kl": 2.236898035334889e-05,
+ "learning_rate": 2.076966784060165e-07,
+ "loss": 0.023751959204673767,
+ "num_tokens": 1054578.0,
+ "reward": 0.4112047851085663,
+ "reward_std": 0.05330020561814308,
+ "rewards/true_env_reward_fn/mean": 0.4112047851085663,
+ "rewards/true_env_reward_fn/std": 0.05330020561814308,
+ "step": 257,
+ "step_time": 4.424114469000415
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 52.125,
+ "completions/mean_terminated_length": 52.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2734522223472595,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2274128645658493,
+ "kl": 5.3426387239596806e-05,
+ "learning_rate": 2.043864031720667e-07,
+ "loss": 0.05165906995534897,
+ "num_tokens": 1060579.0,
+ "reward": 0.2129499912261963,
+ "reward_std": 0.2476053088903427,
+ "rewards/true_env_reward_fn/mean": 0.2129499912261963,
+ "rewards/true_env_reward_fn/std": 0.2476053088903427,
+ "step": 258,
+ "step_time": 3.4830677139998443
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 54.375,
+ "completions/mean_terminated_length": 54.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2013150453567505,
+ "epoch": 2.105691056910569,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20796725153923035,
+ "kl": 8.34841157484334e-05,
+ "learning_rate": 2.0109364331054297e-07,
+ "loss": 0.04251064360141754,
+ "num_tokens": 1065318.0,
+ "reward": 0.2038009911775589,
+ "reward_std": 0.3393669128417969,
+ "rewards/true_env_reward_fn/mean": 0.2038009911775589,
+ "rewards/true_env_reward_fn/std": 0.3393669128417969,
+ "step": 259,
+ "step_time": 3.34712773299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3005307912826538,
+ "epoch": 2.113821138211382,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2342396229505539,
+ "kl": 2.9608699151140172e-05,
+ "learning_rate": 1.9781869365736777e-07,
+ "loss": -0.11542908847332001,
+ "num_tokens": 1068352.0,
+ "reward": 0.5755212306976318,
+ "reward_std": 0.1629202961921692,
+ "rewards/true_env_reward_fn/mean": 0.5755212306976318,
+ "rewards/true_env_reward_fn/std": 0.1629202961921692,
+ "step": 260,
+ "step_time": 3.0931306170004973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1381222009658813,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13169968128204346,
+ "kl": 1.4705466128361877e-05,
+ "learning_rate": 1.9456184745372558e-07,
+ "loss": -0.026440951973199844,
+ "num_tokens": 1070847.0,
+ "reward": 0.5704532265663147,
+ "reward_std": 0.13928835093975067,
+ "rewards/true_env_reward_fn/mean": 0.5704532265663147,
+ "rewards/true_env_reward_fn/std": 0.13928835093975067,
+ "step": 261,
+ "step_time": 3.8980969309996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.241390347480774,
+ "epoch": 2.130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010450400441186503,
+ "kl": 1.6406540453317575e-05,
+ "learning_rate": 1.9132339631980622e-07,
+ "loss": 8.202600838558283e-07,
+ "num_tokens": 1076153.0,
+ "reward": 0.41493332386016846,
+ "reward_std": 0.06228968873620033,
+ "rewards/true_env_reward_fn/mean": 0.41493332386016846,
+ "rewards/true_env_reward_fn/std": 0.06228969246149063,
+ "step": 262,
+ "step_time": 3.6601423579995753
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 81.0,
+ "completions/mean_terminated_length": 81.0,
+ "completions/min_length": 67.0,
+ "completions/min_terminated_length": 67.0,
+ "entropy": 1.1326860189437866,
+ "epoch": 2.138211382113821,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001754833065206185,
+ "kl": 1.4390577689482598e-05,
+ "learning_rate": 1.881036302286923e-07,
+ "loss": 7.2446778176527e-07,
+ "num_tokens": 1080537.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 263,
+ "step_time": 4.190891189999093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.353486955165863,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.215454563498497,
+ "kl": 6.859865607111715e-05,
+ "learning_rate": 1.8490283748039515e-07,
+ "loss": 0.0181141197681427,
+ "num_tokens": 1084056.0,
+ "reward": 0.3049938380718231,
+ "reward_std": 0.4605039656162262,
+ "rewards/true_env_reward_fn/mean": 0.3049938380718231,
+ "rewards/true_env_reward_fn/std": 0.4605039954185486,
+ "step": 264,
+ "step_time": 4.504906432999633
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.375,
+ "completions/mean_terminated_length": 70.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0113105773925781,
+ "epoch": 2.154471544715447,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10072485357522964,
+ "kl": 1.4604076568502933e-05,
+ "learning_rate": 1.8172130467604085e-07,
+ "loss": -0.041721273213624954,
+ "num_tokens": 1090171.0,
+ "reward": 0.47745320200920105,
+ "reward_std": 0.2929421067237854,
+ "rewards/true_env_reward_fn/mean": 0.47745320200920105,
+ "rewards/true_env_reward_fn/std": 0.2929421067237854,
+ "step": 265,
+ "step_time": 6.299696521999067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.375,
+ "completions/mean_terminated_length": 57.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3366597294807434,
+ "epoch": 2.16260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2297964245080948,
+ "kl": 5.677436638507061e-05,
+ "learning_rate": 1.785593166922062e-07,
+ "loss": 0.20361776649951935,
+ "num_tokens": 1094358.0,
+ "reward": 0.3835672438144684,
+ "reward_std": 0.4339357614517212,
+ "rewards/true_env_reward_fn/mean": 0.3835672438144684,
+ "rewards/true_env_reward_fn/std": 0.4339357912540436,
+ "step": 266,
+ "step_time": 4.192992550000781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4040917754173279,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1913405954837799,
+ "kl": 2.2514723241329193e-05,
+ "learning_rate": 1.7541715665541276e-07,
+ "loss": -0.12278837710618973,
+ "num_tokens": 1099685.0,
+ "reward": 0.5316476821899414,
+ "reward_std": 0.1867343932390213,
+ "rewards/true_env_reward_fn/mean": 0.5316476821899414,
+ "rewards/true_env_reward_fn/std": 0.1867344230413437,
+ "step": 267,
+ "step_time": 4.577502725998784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2707499265670776,
+ "epoch": 2.178861788617886,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1252160668373108,
+ "kl": 2.1676510186807718e-05,
+ "learning_rate": 1.722951059167737e-07,
+ "loss": 0.030697450041770935,
+ "num_tokens": 1103244.0,
+ "reward": 0.47949954867362976,
+ "reward_std": 0.10323704034090042,
+ "rewards/true_env_reward_fn/mean": 0.47949954867362976,
+ "rewards/true_env_reward_fn/std": 0.10323705524206161,
+ "step": 268,
+ "step_time": 4.3898782989999745
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3778526186943054,
+ "epoch": 2.186991869918699,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19924476742744446,
+ "kl": 5.189802323002368e-05,
+ "learning_rate": 1.6919344402680231e-07,
+ "loss": 0.03015017881989479,
+ "num_tokens": 1108318.0,
+ "reward": 0.12559716403484344,
+ "reward_std": 0.22310735285282135,
+ "rewards/true_env_reward_fn/mean": 0.12559716403484344,
+ "rewards/true_env_reward_fn/std": 0.22310735285282135,
+ "step": 269,
+ "step_time": 4.379851057999986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2916911840438843,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18381677567958832,
+ "kl": 2.593698127384414e-05,
+ "learning_rate": 1.6611244871038116e-07,
+ "loss": 0.025029506534337997,
+ "num_tokens": 1116025.0,
+ "reward": 0.05200198292732239,
+ "reward_std": 0.276480108499527,
+ "rewards/true_env_reward_fn/mean": 0.05200198292732239,
+ "rewards/true_env_reward_fn/std": 0.276480108499527,
+ "step": 270,
+ "step_time": 3.6788300769985653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 94.625,
+ "completions/mean_terminated_length": 94.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2625537514686584,
+ "epoch": 2.203252032520325,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.09628148376941681,
+ "kl": 2.636932003952097e-05,
+ "learning_rate": 1.6305239584189344e-07,
+ "loss": 0.009904414415359497,
+ "num_tokens": 1120434.0,
+ "reward": 0.5924437046051025,
+ "reward_std": 0.39917245507240295,
+ "rewards/true_env_reward_fn/mean": 0.5924437046051025,
+ "rewards/true_env_reward_fn/std": 0.39917245507240295,
+ "step": 271,
+ "step_time": 9.09279120499923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2491654753684998,
+ "epoch": 2.2113821138211383,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015083610196597874,
+ "kl": 1.4522283436235739e-05,
+ "learning_rate": 1.6001355942052182e-07,
+ "loss": 7.247089683914965e-07,
+ "num_tokens": 1122349.0,
+ "reward": 0.812765896320343,
+ "reward_std": 0.04731824994087219,
+ "rewards/true_env_reward_fn/mean": 0.812765896320343,
+ "rewards/true_env_reward_fn/std": 0.04731824994087219,
+ "step": 272,
+ "step_time": 3.7730076539992297
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1462301015853882,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00012917000276502222,
+ "kl": 1.3649782886204775e-05,
+ "learning_rate": 1.569962115457138e-07,
+ "loss": 6.695274805679219e-07,
+ "num_tokens": 1124831.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 273,
+ "step_time": 3.907510233000721
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2922418713569641,
+ "epoch": 2.227642276422764,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15563185513019562,
+ "kl": 3.604595076467376e-05,
+ "learning_rate": 1.5400062239281858e-07,
+ "loss": -0.034219659864902496,
+ "num_tokens": 1128837.0,
+ "reward": 0.46255773305892944,
+ "reward_std": 0.24893923103809357,
+ "rewards/true_env_reward_fn/mean": 0.46255773305892944,
+ "rewards/true_env_reward_fn/std": 0.24893923103809357,
+ "step": 274,
+ "step_time": 7.502110859999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2709790468215942,
+ "epoch": 2.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24958185851573944,
+ "kl": 6.180045966175385e-05,
+ "learning_rate": 1.5102706018889428e-07,
+ "loss": -0.10148808360099792,
+ "num_tokens": 1132891.0,
+ "reward": 0.4240284562110901,
+ "reward_std": 0.43752968311309814,
+ "rewards/true_env_reward_fn/mean": 0.4240284562110901,
+ "rewards/true_env_reward_fn/std": 0.43752965331077576,
+ "step": 275,
+ "step_time": 3.251475233999372
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2637454867362976,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.439944602083415e-05,
+ "kl": 1.127877567341784e-05,
+ "learning_rate": 1.4807579118869146e-07,
+ "loss": 5.64579522688291e-07,
+ "num_tokens": 1137611.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 276,
+ "step_time": 3.311975311999049
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1065265536308289,
+ "epoch": 2.252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13068579137325287,
+ "kl": 4.4293181417742744e-05,
+ "learning_rate": 1.4514707965081262e-07,
+ "loss": 0.04848391190171242,
+ "num_tokens": 1142087.0,
+ "reward": 0.2852628827095032,
+ "reward_std": 0.23009054362773895,
+ "rewards/true_env_reward_fn/mean": 0.2852628827095032,
+ "rewards/true_env_reward_fn/std": 0.23009057343006134,
+ "step": 277,
+ "step_time": 3.9521008399988204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 104.5,
+ "completions/mean_terminated_length": 104.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5374161005020142,
+ "epoch": 2.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.129844531416893,
+ "kl": 3.9368313082377426e-05,
+ "learning_rate": 1.4224118781404917e-07,
+ "loss": -0.1482687145471573,
+ "num_tokens": 1147731.0,
+ "reward": 0.43388551473617554,
+ "reward_std": 0.39525240659713745,
+ "rewards/true_env_reward_fn/mean": 0.43388551473617554,
+ "rewards/true_env_reward_fn/std": 0.39525243639945984,
+ "step": 278,
+ "step_time": 7.55689369099855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.0975646376609802,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1335010528564453,
+ "kl": 2.2682882445224095e-05,
+ "learning_rate": 1.3935837587390214e-07,
+ "loss": -0.009766265749931335,
+ "num_tokens": 1151246.0,
+ "reward": 0.37655720114707947,
+ "reward_std": 0.37258440256118774,
+ "rewards/true_env_reward_fn/mean": 0.37655720114707947,
+ "rewards/true_env_reward_fn/std": 0.37258440256118774,
+ "step": 279,
+ "step_time": 3.8792882219986495
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1846895217895508,
+ "epoch": 2.2764227642276422,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.138702854514122,
+ "kl": 2.0143882466072682e-05,
+ "learning_rate": 1.3649890195928254e-07,
+ "loss": 0.013748884201049805,
+ "num_tokens": 1153222.0,
+ "reward": 0.8218116760253906,
+ "reward_std": 0.1293545663356781,
+ "rewards/true_env_reward_fn/mean": 0.8218116760253906,
+ "rewards/true_env_reward_fn/std": 0.1293545812368393,
+ "step": 280,
+ "step_time": 3.42550413599929
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 64.0,
+ "completions/mean_terminated_length": 64.0,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3451175689697266,
+ "epoch": 2.2845528455284554,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1508527249097824,
+ "kl": 1.6968931049632374e-05,
+ "learning_rate": 1.336630221093991e-07,
+ "loss": 0.04001428931951523,
+ "num_tokens": 1157606.0,
+ "reward": 0.4994586706161499,
+ "reward_std": 0.10593737661838531,
+ "rewards/true_env_reward_fn/mean": 0.4994586706161499,
+ "rewards/true_env_reward_fn/std": 0.10593737661838531,
+ "step": 281,
+ "step_time": 3.914840199000537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2436452507972717,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12380471080541611,
+ "kl": 4.350653580331709e-05,
+ "learning_rate": 1.3085099025083245e-07,
+ "loss": -0.029160797595977783,
+ "num_tokens": 1162012.0,
+ "reward": 0.5066306591033936,
+ "reward_std": 0.28914663195610046,
+ "rewards/true_env_reward_fn/mean": 0.5066306591033936,
+ "rewards/true_env_reward_fn/std": 0.28914666175842285,
+ "step": 282,
+ "step_time": 3.9093819319987233
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.10621577501297,
+ "epoch": 2.3008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001239393459400162,
+ "kl": 1.6069413049990544e-05,
+ "learning_rate": 1.2806305817479771e-07,
+ "loss": 8.174432082341809e-07,
+ "num_tokens": 1165367.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 283,
+ "step_time": 3.8117841049988783
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 66.0,
+ "completions/mean_terminated_length": 66.0,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.382466197013855,
+ "epoch": 2.3089430894308944,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1446879506111145,
+ "kl": 3.576014205464162e-05,
+ "learning_rate": 1.2529947551459964e-07,
+ "loss": -0.04731176793575287,
+ "num_tokens": 1169243.0,
+ "reward": 0.496622234582901,
+ "reward_std": 0.17860308289527893,
+ "rewards/true_env_reward_fn/mean": 0.496622234582901,
+ "rewards/true_env_reward_fn/std": 0.17860306799411774,
+ "step": 284,
+ "step_time": 3.649135475998264
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1946157813072205,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10612626373767853,
+ "kl": 2.0257402866263874e-05,
+ "learning_rate": 1.2256048972327967e-07,
+ "loss": -0.029511645436286926,
+ "num_tokens": 1173594.0,
+ "reward": 0.5235810279846191,
+ "reward_std": 0.2810492217540741,
+ "rewards/true_env_reward_fn/mean": 0.5235810279846191,
+ "rewards/true_env_reward_fn/std": 0.2810492217540741,
+ "step": 285,
+ "step_time": 6.7011265000001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2879216074943542,
+ "epoch": 2.3252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12234477698802948,
+ "kl": 1.4488860415440286e-05,
+ "learning_rate": 1.1984634605145978e-07,
+ "loss": -0.05451745539903641,
+ "num_tokens": 1176384.0,
+ "reward": 0.6496104001998901,
+ "reward_std": 0.3368098735809326,
+ "rewards/true_env_reward_fn/mean": 0.6496104001998901,
+ "rewards/true_env_reward_fn/std": 0.3368098735809326,
+ "step": 286,
+ "step_time": 4.18911992899848
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0649144053459167,
+ "epoch": 2.3333333333333335,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1467445194721222,
+ "kl": 3.947542245441582e-05,
+ "learning_rate": 1.17157287525381e-07,
+ "loss": -0.029090911149978638,
+ "num_tokens": 1179651.0,
+ "reward": 0.4754716753959656,
+ "reward_std": 0.2500669062137604,
+ "rewards/true_env_reward_fn/mean": 0.4754716753959656,
+ "rewards/true_env_reward_fn/std": 0.2500669062137604,
+ "step": 287,
+ "step_time": 4.5810332049986755
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.002779871225357,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12533478438854218,
+ "kl": 2.3054610210238025e-05,
+ "learning_rate": 1.1449355492514437e-07,
+ "loss": -0.04500773549079895,
+ "num_tokens": 1181547.0,
+ "reward": 0.7334807515144348,
+ "reward_std": 0.12557923793792725,
+ "rewards/true_env_reward_fn/mean": 0.7334807515144348,
+ "rewards/true_env_reward_fn/std": 0.12557923793792725,
+ "step": 288,
+ "step_time": 3.8661079009998502
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 81.25,
+ "completions/mean_terminated_length": 81.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2280530333518982,
+ "epoch": 2.3495934959349594,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23682235181331635,
+ "kl": 5.290110129863024e-05,
+ "learning_rate": 1.1185538676315052e-07,
+ "loss": 0.5538168549537659,
+ "num_tokens": 1183973.0,
+ "reward": 0.5785378217697144,
+ "reward_std": 0.29215970635414124,
+ "rewards/true_env_reward_fn/mean": 0.5785378217697144,
+ "rewards/true_env_reward_fn/std": 0.2921597361564636,
+ "step": 289,
+ "step_time": 11.744046860998424
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.264986515045166,
+ "epoch": 2.3577235772357725,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001335782726528123,
+ "kl": 1.7416054106433876e-05,
+ "learning_rate": 1.0924301926274248e-07,
+ "loss": 8.602528396295384e-07,
+ "num_tokens": 1187311.0,
+ "reward": 0.4899469017982483,
+ "reward_std": 0.24049179255962372,
+ "rewards/true_env_reward_fn/mean": 0.4899469017982483,
+ "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "step": 290,
+ "step_time": 4.003069795000556
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.222926139831543,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.209110481198877e-05,
+ "kl": 1.1452370472397888e-05,
+ "learning_rate": 1.0665668633705572e-07,
+ "loss": 5.764911747974111e-07,
+ "num_tokens": 1191370.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 291,
+ "step_time": 4.77749846199913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.313057780265808,
+ "epoch": 2.3739837398373984,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.671779556199908e-05,
+ "kl": 1.1384066056052689e-05,
+ "learning_rate": 1.0409661956807174e-07,
+ "loss": 5.689008162335085e-07,
+ "num_tokens": 1195851.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 292,
+ "step_time": 3.4948791150000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3050158619880676,
+ "epoch": 2.3821138211382116,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15694627165794373,
+ "kl": 2.6891467314271722e-05,
+ "learning_rate": 1.0156304818588308e-07,
+ "loss": -0.044491954147815704,
+ "num_tokens": 1201316.0,
+ "reward": 0.18382371962070465,
+ "reward_std": 0.18414245545864105,
+ "rewards/true_env_reward_fn/mean": 0.18382371962070465,
+ "rewards/true_env_reward_fn/std": 0.18414245545864105,
+ "step": 293,
+ "step_time": 3.900356202000694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2079132199287415,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25304585695266724,
+ "kl": 4.250183792464668e-05,
+ "learning_rate": 9.905619904816749e-08,
+ "loss": 0.008140146732330322,
+ "num_tokens": 1204090.0,
+ "reward": 0.43626630306243896,
+ "reward_std": 0.3027261197566986,
+ "rewards/true_env_reward_fn/mean": 0.43626630306243896,
+ "rewards/true_env_reward_fn/std": 0.3027261197566986,
+ "step": 294,
+ "step_time": 3.1466946830005327
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2324314713478088,
+ "epoch": 2.3983739837398375,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011851672024931759,
+ "kl": 1.4868088328512385e-05,
+ "learning_rate": 9.657629661987531e-08,
+ "loss": 7.534490578109398e-07,
+ "num_tokens": 1206449.0,
+ "reward": 0.7232838273048401,
+ "reward_std": 0.008955853059887886,
+ "rewards/true_env_reward_fn/mean": 0.7232838273048401,
+ "rewards/true_env_reward_fn/std": 0.008955853059887886,
+ "step": 295,
+ "step_time": 3.5492840760016406
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 65.125,
+ "completions/mean_terminated_length": 65.125,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.2110244631767273,
+ "epoch": 2.40650406504065,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18513579666614532,
+ "kl": 6.467000457632821e-05,
+ "learning_rate": 9.412356295313019e-08,
+ "loss": -0.008508354425430298,
+ "num_tokens": 1211294.0,
+ "reward": 0.38126999139785767,
+ "reward_std": 0.22339044511318207,
+ "rewards/true_env_reward_fn/mean": 0.38126999139785767,
+ "rewards/true_env_reward_fn/std": 0.22339043021202087,
+ "step": 296,
+ "step_time": 3.6129159619995335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1828523874282837,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13915051519870758,
+ "kl": 2.76857699645916e-05,
+ "learning_rate": 9.169821766734668e-08,
+ "loss": -0.019797056913375854,
+ "num_tokens": 1214972.0,
+ "reward": 0.5778937339782715,
+ "reward_std": 0.4268997013568878,
+ "rewards/true_env_reward_fn/mean": 0.5778937339782715,
+ "rewards/true_env_reward_fn/std": 0.4268997013568878,
+ "step": 297,
+ "step_time": 3.5299333029997797
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2796242237091064,
+ "epoch": 2.4227642276422765,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23225097358226776,
+ "kl": 5.458398300106637e-05,
+ "learning_rate": 8.930047792956585e-08,
+ "loss": 0.028596192598342896,
+ "num_tokens": 1221117.0,
+ "reward": 0.09695600718259811,
+ "reward_std": 0.23755072057247162,
+ "rewards/true_env_reward_fn/mean": 0.09695600718259811,
+ "rewards/true_env_reward_fn/std": 0.23755072057247162,
+ "step": 298,
+ "step_time": 3.9654863289997593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3131609559059143,
+ "epoch": 2.430894308943089,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15491950511932373,
+ "kl": 2.3678386241954286e-05,
+ "learning_rate": 8.693055843500867e-08,
+ "loss": 0.08264091610908508,
+ "num_tokens": 1226670.0,
+ "reward": 0.22746901214122772,
+ "reward_std": 0.2765822112560272,
+ "rewards/true_env_reward_fn/mean": 0.22746901214122772,
+ "rewards/true_env_reward_fn/std": 0.2765822112560272,
+ "step": 299,
+ "step_time": 3.9300464680000005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 61.75,
+ "completions/mean_terminated_length": 61.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.113481342792511,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14261895418167114,
+ "kl": 2.8488028874562588e-05,
+ "learning_rate": 8.458867138785369e-08,
+ "loss": 0.012870386242866516,
+ "num_tokens": 1230460.0,
+ "reward": 0.6178936958312988,
+ "reward_std": 0.3830615282058716,
+ "rewards/true_env_reward_fn/mean": 0.6178936958312988,
+ "rewards/true_env_reward_fn/std": 0.3830614984035492,
+ "step": 300,
+ "step_time": 4.004705740000645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2633118629455566,
+ "epoch": 2.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1559160053730011,
+ "kl": 3.0400691230170196e-05,
+ "learning_rate": 8.227502648223494e-08,
+ "loss": 0.023099511861801147,
+ "num_tokens": 1234850.0,
+ "reward": 0.3328326642513275,
+ "reward_std": 0.2606535255908966,
+ "rewards/true_env_reward_fn/mean": 0.3328326642513275,
+ "rewards/true_env_reward_fn/std": 0.260653555393219,
+ "step": 301,
+ "step_time": 4.035395368000536
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 79.875,
+ "completions/mean_terminated_length": 79.875,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4545465111732483,
+ "epoch": 2.4552845528455283,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13911883533000946,
+ "kl": 5.561073703574948e-05,
+ "learning_rate": 7.99898308834662e-08,
+ "loss": 0.10435273498296738,
+ "num_tokens": 1237629.0,
+ "reward": 0.3929310441017151,
+ "reward_std": 0.294041246175766,
+ "rewards/true_env_reward_fn/mean": 0.3929310441017151,
+ "rewards/true_env_reward_fn/std": 0.294041246175766,
+ "step": 302,
+ "step_time": 6.382147416999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.4197523593902588,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11274738609790802,
+ "kl": 1.6755700926296413e-05,
+ "learning_rate": 7.773328920949151e-08,
+ "loss": -0.1040230244398117,
+ "num_tokens": 1240738.0,
+ "reward": 0.6045305728912354,
+ "reward_std": 0.10337947309017181,
+ "rewards/true_env_reward_fn/mean": 0.6045305728912354,
+ "rewards/true_env_reward_fn/std": 0.10337948054075241,
+ "step": 303,
+ "step_time": 3.5754013399982796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 69.625,
+ "completions/mean_terminated_length": 69.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3458821177482605,
+ "epoch": 2.4715447154471546,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11526551097631454,
+ "kl": 2.5664294298621826e-05,
+ "learning_rate": 7.550560351256309e-08,
+ "loss": 0.0017936527729034424,
+ "num_tokens": 1245803.0,
+ "reward": 0.2295326590538025,
+ "reward_std": 0.3014034032821655,
+ "rewards/true_env_reward_fn/mean": 0.2295326590538025,
+ "rewards/true_env_reward_fn/std": 0.3014034032821655,
+ "step": 304,
+ "step_time": 4.8661928239998815
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2587900161743164,
+ "epoch": 2.4796747967479673,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1804642677307129,
+ "kl": 3.644101343525108e-05,
+ "learning_rate": 7.330697326114972e-08,
+ "loss": 0.006622403860092163,
+ "num_tokens": 1250224.0,
+ "reward": 0.392159104347229,
+ "reward_std": 0.23998720943927765,
+ "rewards/true_env_reward_fn/mean": 0.392159104347229,
+ "rewards/true_env_reward_fn/std": 0.23998722434043884,
+ "step": 305,
+ "step_time": 4.381737805999364
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 88.75,
+ "completions/mean_terminated_length": 88.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.235496699810028,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010392541298642755,
+ "kl": 1.9250182049290743e-05,
+ "learning_rate": 7.113759532207599e-08,
+ "loss": 9.166102472590865e-07,
+ "num_tokens": 1256018.0,
+ "reward": 0.45198333263397217,
+ "reward_std": 0.0026191486977040768,
+ "rewards/true_env_reward_fn/mean": 0.45198333263397217,
+ "rewards/true_env_reward_fn/std": 0.0026191489305347204,
+ "step": 306,
+ "step_time": 6.6232522029986285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1592004895210266,
+ "epoch": 2.4959349593495936,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.044789915904403e-05,
+ "kl": 9.38343055167934e-06,
+ "learning_rate": 6.899766394289516e-08,
+ "loss": 4.709004315373022e-07,
+ "num_tokens": 1259980.0,
+ "reward": 0.4911326766014099,
+ "reward_std": 0.01917082816362381,
+ "rewards/true_env_reward_fn/mean": 0.4911326766014099,
+ "rewards/true_env_reward_fn/std": 0.01917083002626896,
+ "step": 307,
+ "step_time": 3.398790989002009
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1962818503379822,
+ "epoch": 2.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12975044548511505,
+ "kl": 9.622429843147984e-06,
+ "learning_rate": 6.688737073449563e-08,
+ "loss": -0.028225116431713104,
+ "num_tokens": 1265125.0,
+ "reward": 0.3520139753818512,
+ "reward_std": 0.4090423583984375,
+ "rewards/true_env_reward_fn/mean": 0.3520139753818512,
+ "rewards/true_env_reward_fn/std": 0.4090423583984375,
+ "step": 308,
+ "step_time": 3.4128740100004507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1477364301681519,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.428078242810443e-05,
+ "kl": 1.2252480246388586e-05,
+ "learning_rate": 6.480690465394398e-08,
+ "loss": 6.147511157905683e-07,
+ "num_tokens": 1269679.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 309,
+ "step_time": 3.4761773770005675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.1964089274406433,
+ "epoch": 2.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010435594595037401,
+ "kl": 1.349770900560543e-05,
+ "learning_rate": 6.275645198756629e-08,
+ "loss": 6.71000861984794e-07,
+ "num_tokens": 1272836.0,
+ "reward": 0.4899469017982483,
+ "reward_std": 0.24049179255962372,
+ "rewards/true_env_reward_fn/mean": 0.4899469017982483,
+ "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "step": 310,
+ "step_time": 3.522990450999714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 399.0,
+ "completions/max_terminated_length": 399.0,
+ "completions/mean_length": 101.125,
+ "completions/mean_terminated_length": 101.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.428375780582428,
+ "epoch": 2.5284552845528454,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10063836723566055,
+ "kl": 2.3114703708415618e-05,
+ "learning_rate": 6.073619633426675e-08,
+ "loss": -0.1970823109149933,
+ "num_tokens": 1279065.0,
+ "reward": 0.4598180055618286,
+ "reward_std": 0.23299095034599304,
+ "rewards/true_env_reward_fn/mean": 0.4598180055618286,
+ "rewards/true_env_reward_fn/std": 0.23299095034599304,
+ "step": 311,
+ "step_time": 16.64716850199875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 62.875,
+ "completions/mean_terminated_length": 62.875,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0843100249767303,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015719492512289435,
+ "kl": 1.4557146187144099e-05,
+ "learning_rate": 5.8746318589089337e-08,
+ "loss": 7.280061709025176e-07,
+ "num_tokens": 1280592.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 312,
+ "step_time": 3.2888442910007143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 72.0,
+ "completions/mean_terminated_length": 72.0,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.240959882736206,
+ "epoch": 2.5447154471544717,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12602119147777557,
+ "kl": 1.7604435015527997e-05,
+ "learning_rate": 5.67869969270196e-08,
+ "loss": 9.685754776000977e-07,
+ "num_tokens": 1284772.0,
+ "reward": 0.4882529377937317,
+ "reward_std": 0.2615475356578827,
+ "rewards/true_env_reward_fn/mean": 0.4882529377937317,
+ "rewards/true_env_reward_fn/std": 0.2615475356578827,
+ "step": 313,
+ "step_time": 4.0729800409990276
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1718184351921082,
+ "epoch": 2.5528455284552845,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14716969430446625,
+ "kl": 1.3087485967844259e-05,
+ "learning_rate": 5.4858406787030845e-08,
+ "loss": 0.016231566667556763,
+ "num_tokens": 1287839.0,
+ "reward": 0.6037359237670898,
+ "reward_std": 0.10266375541687012,
+ "rewards/true_env_reward_fn/mean": 0.6037359237670898,
+ "rewards/true_env_reward_fn/std": 0.10266375541687012,
+ "step": 314,
+ "step_time": 3.175157601999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 40.5,
+ "completions/mean_terminated_length": 40.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3117390871047974,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2794886827468872,
+ "kl": 4.65317443740787e-05,
+ "learning_rate": 5.29607208563756e-08,
+ "loss": -0.09303665161132812,
+ "num_tokens": 1291515.0,
+ "reward": 0.5445280075073242,
+ "reward_std": 0.19415061175823212,
+ "rewards/true_env_reward_fn/mean": 0.5445280075073242,
+ "rewards/true_env_reward_fn/std": 0.19415059685707092,
+ "step": 315,
+ "step_time": 2.636586960999921
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.15413236618042,
+ "epoch": 2.569105691056911,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22290591895580292,
+ "kl": 6.188569841469871e-05,
+ "learning_rate": 5.10941090551225e-08,
+ "loss": -0.013920806348323822,
+ "num_tokens": 1296399.0,
+ "reward": 0.3418610095977783,
+ "reward_std": 0.29834023118019104,
+ "rewards/true_env_reward_fn/mean": 0.3418610095977783,
+ "rewards/true_env_reward_fn/std": 0.2983402609825134,
+ "step": 316,
+ "step_time": 3.7106533750011295
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 102.875,
+ "completions/mean_terminated_length": 102.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.2713149785995483,
+ "epoch": 2.5772357723577235,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11319029331207275,
+ "kl": 1.3128932096151402e-05,
+ "learning_rate": 4.9258738520942025e-08,
+ "loss": 0.04848237335681915,
+ "num_tokens": 1301626.0,
+ "reward": 0.4802166819572449,
+ "reward_std": 0.05550921708345413,
+ "rewards/true_env_reward_fn/mean": 0.4802166819572449,
+ "rewards/true_env_reward_fn/std": 0.05550922453403473,
+ "step": 317,
+ "step_time": 11.140286670997739
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 45.625,
+ "completions/mean_terminated_length": 45.625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 0.9857950508594513,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011810586147475988,
+ "kl": 1.2108953342249151e-05,
+ "learning_rate": 4.745477359414045e-08,
+ "loss": 6.04832052886195e-07,
+ "num_tokens": 1303619.0,
+ "reward": 0.6204532384872437,
+ "reward_std": 0.10097470134496689,
+ "rewards/true_env_reward_fn/mean": 0.6204532384872437,
+ "rewards/true_env_reward_fn/std": 0.10097470879554749,
+ "step": 318,
+ "step_time": 2.977002043000539
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2560631036758423,
+ "epoch": 2.59349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24155104160308838,
+ "kl": 3.4935148505610414e-05,
+ "learning_rate": 4.5682375802945295e-08,
+ "loss": 0.1817030906677246,
+ "num_tokens": 1310682.0,
+ "reward": 0.055160000920295715,
+ "reward_std": 0.13755998015403748,
+ "rewards/true_env_reward_fn/mean": 0.055160000920295715,
+ "rewards/true_env_reward_fn/std": 0.13755998015403748,
+ "step": 319,
+ "step_time": 4.712957282001298
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 69.75,
+ "completions/mean_terminated_length": 69.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2313570380210876,
+ "epoch": 2.6016260162601625,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011636545968940482,
+ "kl": 1.566714945511194e-05,
+ "learning_rate": 4.394170384904137e-08,
+ "loss": 7.867492968216538e-07,
+ "num_tokens": 1315436.0,
+ "reward": 0.4662666618824005,
+ "reward_std": 0.007412042934447527,
+ "rewards/true_env_reward_fn/mean": 0.4662666618824005,
+ "rewards/true_env_reward_fn/std": 0.007412043400108814,
+ "step": 320,
+ "step_time": 4.536037327998201
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 49.625,
+ "completions/mean_terminated_length": 49.625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.0671695470809937,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00013134608161635697,
+ "kl": 1.7747691344993655e-05,
+ "learning_rate": 4.223291359336074e-08,
+ "loss": 8.864202527547604e-07,
+ "num_tokens": 1318837.0,
+ "reward": 0.6651140451431274,
+ "reward_std": 0.20516473054885864,
+ "rewards/true_env_reward_fn/mean": 0.6651140451431274,
+ "rewards/true_env_reward_fn/std": 0.20516474545001984,
+ "step": 321,
+ "step_time": 2.9819667609990574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3910706639289856,
+ "epoch": 2.617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1727662831544876,
+ "kl": 1.9147763850924093e-05,
+ "learning_rate": 4.05561580421272e-08,
+ "loss": 0.07752666622400284,
+ "num_tokens": 1324515.0,
+ "reward": 0.43344998359680176,
+ "reward_std": 0.039401449263095856,
+ "rewards/true_env_reward_fn/mean": 0.43344998359680176,
+ "rewards/true_env_reward_fn/std": 0.039401449263095856,
+ "step": 322,
+ "step_time": 7.645187717998851
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 65.125,
+ "completions/mean_terminated_length": 65.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1752581596374512,
+ "epoch": 2.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14819224178791046,
+ "kl": 1.1951405213039834e-05,
+ "learning_rate": 3.891158733315505e-08,
+ "loss": 0.0940035805106163,
+ "num_tokens": 1327268.0,
+ "reward": 0.6361198425292969,
+ "reward_std": 0.13543039560317993,
+ "rewards/true_env_reward_fn/mean": 0.6361198425292969,
+ "rewards/true_env_reward_fn/std": 0.13543038070201874,
+ "step": 323,
+ "step_time": 4.102753150000353
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.75,
+ "completions/mean_terminated_length": 69.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.4734079241752625,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21136029064655304,
+ "kl": 4.428037027537357e-05,
+ "learning_rate": 3.7299348722406965e-08,
+ "loss": -0.13074398040771484,
+ "num_tokens": 1331614.0,
+ "reward": 0.3051303029060364,
+ "reward_std": 0.20326493680477142,
+ "rewards/true_env_reward_fn/mean": 0.3051303029060364,
+ "rewards/true_env_reward_fn/std": 0.20326495170593262,
+ "step": 324,
+ "step_time": 5.413495869999679
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 78.625,
+ "completions/mean_terminated_length": 78.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.5631850361824036,
+ "epoch": 2.642276422764228,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18741382658481598,
+ "kl": 3.721813300217036e-05,
+ "learning_rate": 3.57195865708074e-08,
+ "loss": 0.20410414040088654,
+ "num_tokens": 1334991.0,
+ "reward": 0.4402502775192261,
+ "reward_std": 0.27313899993896484,
+ "rewards/true_env_reward_fn/mean": 0.4402502775192261,
+ "rewards/true_env_reward_fn/std": 0.27313899993896484,
+ "step": 325,
+ "step_time": 7.439067015000546
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2137504816055298,
+ "epoch": 2.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1392536610364914,
+ "kl": 2.5514624212519266e-05,
+ "learning_rate": 3.417244233131709e-08,
+ "loss": -0.07728786766529083,
+ "num_tokens": 1340886.0,
+ "reward": 0.3032499849796295,
+ "reward_std": 0.24293993413448334,
+ "rewards/true_env_reward_fn/mean": 0.3032499849796295,
+ "rewards/true_env_reward_fn/std": 0.24293994903564453,
+ "step": 326,
+ "step_time": 3.6837621579998086
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 45.125,
+ "completions/mean_terminated_length": 45.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1947113871574402,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23312504589557648,
+ "kl": 6.829304766142741e-05,
+ "learning_rate": 3.265805453626713e-08,
+ "loss": 0.08558641374111176,
+ "num_tokens": 1344823.0,
+ "reward": 0.26869305968284607,
+ "reward_std": 0.30387282371520996,
+ "rewards/true_env_reward_fn/mean": 0.26869305968284607,
+ "rewards/true_env_reward_fn/std": 0.30387285351753235,
+ "step": 327,
+ "step_time": 3.26592919200084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 70.125,
+ "completions/mean_terminated_length": 70.125,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1701073944568634,
+ "epoch": 2.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12238985300064087,
+ "kl": 1.6267454157059547e-05,
+ "learning_rate": 3.117655878495458e-08,
+ "loss": -0.024053269997239113,
+ "num_tokens": 1348012.0,
+ "reward": 0.5990839004516602,
+ "reward_std": 0.21510076522827148,
+ "rewards/true_env_reward_fn/mean": 0.5990839004516602,
+ "rewards/true_env_reward_fn/std": 0.21510076522827148,
+ "step": 328,
+ "step_time": 4.166060536001169
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.75,
+ "completions/mean_terminated_length": 49.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3588789701461792,
+ "epoch": 2.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21684816479682922,
+ "kl": 0.00012067620264133438,
+ "learning_rate": 2.9728087731500884e-08,
+ "loss": 0.017311420291662216,
+ "num_tokens": 1352082.0,
+ "reward": 0.15680472552776337,
+ "reward_std": 0.3331984877586365,
+ "rewards/true_env_reward_fn/mean": 0.15680472552776337,
+ "rewards/true_env_reward_fn/std": 0.3331984877586365,
+ "step": 329,
+ "step_time": 2.947034541000903
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 57.125,
+ "completions/mean_terminated_length": 57.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2207121849060059,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14501330256462097,
+ "kl": 3.47198892995948e-05,
+ "learning_rate": 2.831277107297394e-08,
+ "loss": -0.009820610284805298,
+ "num_tokens": 1356867.0,
+ "reward": 0.508467972278595,
+ "reward_std": 0.10515833646059036,
+ "rewards/true_env_reward_fn/mean": 0.508467972278595,
+ "rewards/true_env_reward_fn/std": 0.10515833646059036,
+ "step": 330,
+ "step_time": 3.278030399000272
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 71.5,
+ "completions/mean_terminated_length": 71.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2053698897361755,
+ "epoch": 2.6910569105691056,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1709476113319397,
+ "kl": 1.9107792468275875e-05,
+ "learning_rate": 2.693073553777494e-08,
+ "loss": 0.04886455833911896,
+ "num_tokens": 1361747.0,
+ "reward": 0.3092518746852875,
+ "reward_std": 0.40765801072120667,
+ "rewards/true_env_reward_fn/mean": 0.3092518746852875,
+ "rewards/true_env_reward_fn/std": 0.40765801072120667,
+ "step": 331,
+ "step_time": 5.705391269999382
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 70.625,
+ "completions/mean_terminated_length": 70.625,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.3019116520881653,
+ "epoch": 2.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13118988275527954,
+ "kl": 6.723498518113047e-05,
+ "learning_rate": 2.5582104874290888e-08,
+ "loss": 0.04069610685110092,
+ "num_tokens": 1366792.0,
+ "reward": 0.39014729857444763,
+ "reward_std": 0.4048542380332947,
+ "rewards/true_env_reward_fn/mean": 0.39014729857444763,
+ "rewards/true_env_reward_fn/std": 0.40485426783561707,
+ "step": 332,
+ "step_time": 4.941442722001739
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 122.25,
+ "completions/mean_terminated_length": 66.5714340209961,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8574695587158203,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08299780637025833,
+ "kl": 8.204095138353296e-06,
+ "learning_rate": 2.4266999839814484e-08,
+ "loss": -0.5517755150794983,
+ "num_tokens": 1373066.0,
+ "reward": 0.19574999809265137,
+ "reward_std": 0.30308666825294495,
+ "rewards/true_env_reward_fn/mean": 0.19574999809265137,
+ "rewards/true_env_reward_fn/std": 0.30308666825294495,
+ "step": 333,
+ "step_time": 20.7688721859995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 49.625,
+ "completions/mean_terminated_length": 49.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.191849946975708,
+ "epoch": 2.7154471544715446,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001034519009408541,
+ "kl": 1.4263560842664447e-05,
+ "learning_rate": 2.2985538189730946e-08,
+ "loss": 7.244366315717343e-07,
+ "num_tokens": 1375815.0,
+ "reward": 0.6581807136535645,
+ "reward_std": 0.21257677674293518,
+ "rewards/true_env_reward_fn/mean": 0.6581807136535645,
+ "rewards/true_env_reward_fn/std": 0.21257677674293518,
+ "step": 334,
+ "step_time": 3.5656982629971026
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 59.75,
+ "completions/mean_terminated_length": 59.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2259244322776794,
+ "epoch": 2.7235772357723578,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12885747849941254,
+ "kl": 4.455958514881786e-05,
+ "learning_rate": 2.1737834666974276e-08,
+ "loss": -0.06481382250785828,
+ "num_tokens": 1379933.0,
+ "reward": 0.31903696060180664,
+ "reward_std": 0.1877410113811493,
+ "rewards/true_env_reward_fn/mean": 0.31903696060180664,
+ "rewards/true_env_reward_fn/std": 0.18774102628231049,
+ "step": 335,
+ "step_time": 3.712153099000716
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 45.0,
+ "completions/mean_terminated_length": 45.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2184038758277893,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.258829265832901,
+ "kl": 6.149329965410288e-05,
+ "learning_rate": 2.0524000991753465e-08,
+ "loss": -0.04611395671963692,
+ "num_tokens": 1383205.0,
+ "reward": 0.41517797112464905,
+ "reward_std": 0.33201757073402405,
+ "rewards/true_env_reward_fn/mean": 0.41517797112464905,
+ "rewards/true_env_reward_fn/std": 0.33201757073402405,
+ "step": 336,
+ "step_time": 3.02953305399933
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 206.0,
+ "completions/max_terminated_length": 206.0,
+ "completions/mean_length": 87.875,
+ "completions/mean_terminated_length": 87.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2387716174125671,
+ "epoch": 2.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16021011769771576,
+ "kl": 1.6921320366236614e-05,
+ "learning_rate": 1.9344145851548243e-08,
+ "loss": -0.14579075574874878,
+ "num_tokens": 1389136.0,
+ "reward": 0.4916413426399231,
+ "reward_std": 0.22573381662368774,
+ "rewards/true_env_reward_fn/mean": 0.4916413426399231,
+ "rewards/true_env_reward_fn/std": 0.22573381662368774,
+ "step": 337,
+ "step_time": 9.208789625999998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2942972779273987,
+ "epoch": 2.747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2804953157901764,
+ "kl": 6.65090201437124e-05,
+ "learning_rate": 1.8198374891377966e-08,
+ "loss": 0.14034238457679749,
+ "num_tokens": 1392442.0,
+ "reward": 0.458748459815979,
+ "reward_std": 0.24402853846549988,
+ "rewards/true_env_reward_fn/mean": 0.458748459815979,
+ "rewards/true_env_reward_fn/std": 0.24402853846549988,
+ "step": 338,
+ "step_time": 3.383874344001015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.433261752128601,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11875848472118378,
+ "kl": 2.3840807443775702e-05,
+ "learning_rate": 1.7086790704341402e-08,
+ "loss": -0.015734028071165085,
+ "num_tokens": 1396235.0,
+ "reward": 0.3571140468120575,
+ "reward_std": 0.10167346149682999,
+ "rewards/true_env_reward_fn/mean": 0.3571140468120575,
+ "rewards/true_env_reward_fn/std": 0.10167345404624939,
+ "step": 339,
+ "step_time": 4.3859739109993825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 97.75,
+ "completions/mean_terminated_length": 97.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.5857495069503784,
+ "epoch": 2.7642276422764227,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.698282959405333e-05,
+ "kl": 1.4666956758446759e-05,
+ "learning_rate": 1.600949282243089e-08,
+ "loss": 7.313840342249023e-07,
+ "num_tokens": 1401073.0,
+ "reward": 0.3621603846549988,
+ "reward_std": 0.10388225317001343,
+ "rewards/true_env_reward_fn/mean": 0.3621603846549988,
+ "rewards/true_env_reward_fn/std": 0.10388225317001343,
+ "step": 340,
+ "step_time": 7.751754675999109
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0097725987434387,
+ "epoch": 2.772357723577236,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011211638775421306,
+ "kl": 1.3342013517103624e-05,
+ "learning_rate": 1.496657770762013e-08,
+ "loss": 6.635225417994661e-07,
+ "num_tokens": 1403921.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 341,
+ "step_time": 3.4684136910000234
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1732905507087708,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14162735641002655,
+ "kl": 4.145995080762077e-05,
+ "learning_rate": 1.3958138743226688e-08,
+ "loss": 0.007000558078289032,
+ "num_tokens": 1407681.0,
+ "reward": 0.5386021733283997,
+ "reward_std": 0.31964072585105896,
+ "rewards/true_env_reward_fn/mean": 0.5386021733283997,
+ "rewards/true_env_reward_fn/std": 0.31964072585105896,
+ "step": 342,
+ "step_time": 3.5975158429992007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0845041275024414,
+ "epoch": 2.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0979820266366005,
+ "kl": 1.916076598718064e-05,
+ "learning_rate": 1.29842662255506e-08,
+ "loss": -0.049884289503097534,
+ "num_tokens": 1413182.0,
+ "reward": 0.316629558801651,
+ "reward_std": 0.16504530608654022,
+ "rewards/true_env_reward_fn/mean": 0.316629558801651,
+ "rewards/true_env_reward_fn/std": 0.16504532098770142,
+ "step": 343,
+ "step_time": 4.6961259659983625
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.1820043921470642,
+ "epoch": 2.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1726708859205246,
+ "kl": 2.1745819140051026e-05,
+ "learning_rate": 1.2045047355789107e-08,
+ "loss": 0.048721786588430405,
+ "num_tokens": 1415472.0,
+ "reward": 0.6344382762908936,
+ "reward_std": 0.07378704100847244,
+ "rewards/true_env_reward_fn/mean": 0.6344382762908936,
+ "rewards/true_env_reward_fn/std": 0.07378704845905304,
+ "step": 344,
+ "step_time": 3.2691068980002456
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0933732390403748,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21739810705184937,
+ "kl": 2.0197898265905678e-05,
+ "learning_rate": 1.1140566232228677e-08,
+ "loss": 0.0031427331268787384,
+ "num_tokens": 1417385.0,
+ "reward": 0.5309104323387146,
+ "reward_std": 0.18906596302986145,
+ "rewards/true_env_reward_fn/mean": 0.5309104323387146,
+ "rewards/true_env_reward_fn/std": 0.18906594812870026,
+ "step": 345,
+ "step_time": 3.9515345389991126
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2968629598617554,
+ "epoch": 2.813008130081301,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20110434293746948,
+ "kl": 4.813342457055114e-05,
+ "learning_rate": 1.0270903842714673e-08,
+ "loss": 0.062390223145484924,
+ "num_tokens": 1423496.0,
+ "reward": 0.14946666359901428,
+ "reward_std": 0.24863645434379578,
+ "rewards/true_env_reward_fn/mean": 0.14946666359901428,
+ "rewards/true_env_reward_fn/std": 0.24863646924495697,
+ "step": 346,
+ "step_time": 4.331638548001138
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.153487741947174,
+ "epoch": 2.821138211382114,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011695526336552575,
+ "kl": 1.7154279703390785e-05,
+ "learning_rate": 9.436138057399601e-09,
+ "loss": 8.606326673543663e-07,
+ "num_tokens": 1427182.0,
+ "reward": 0.5776733160018921,
+ "reward_std": 0.11168669909238815,
+ "rewards/true_env_reward_fn/mean": 0.5776733160018921,
+ "rewards/true_env_reward_fn/std": 0.11168669164180756,
+ "step": 347,
+ "step_time": 3.416529175998221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3210760354995728,
+ "epoch": 2.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13752248883247375,
+ "kl": 3.656550507002976e-05,
+ "learning_rate": 8.636343621770948e-09,
+ "loss": -0.029253121465444565,
+ "num_tokens": 1431839.0,
+ "reward": 0.47378063201904297,
+ "reward_std": 0.32937321066856384,
+ "rewards/true_env_reward_fn/mean": 0.47378063201904297,
+ "rewards/true_env_reward_fn/std": 0.32937324047088623,
+ "step": 348,
+ "step_time": 3.957806388998506
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 83.75,
+ "completions/mean_terminated_length": 83.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.3464251160621643,
+ "epoch": 2.83739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11163587868213654,
+ "kl": 1.8057865418086294e-05,
+ "learning_rate": 7.87159214995774e-09,
+ "loss": 0.01491672545671463,
+ "num_tokens": 1434785.0,
+ "reward": 0.6471105217933655,
+ "reward_std": 0.1651550978422165,
+ "rewards/true_env_reward_fn/mean": 0.6471105217933655,
+ "rewards/true_env_reward_fn/std": 0.1651550978422165,
+ "step": 349,
+ "step_time": 5.513293031999638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.0,
+ "completions/mean_terminated_length": 57.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1192307472229004,
+ "epoch": 2.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20383992791175842,
+ "kl": 1.563574414831237e-05,
+ "learning_rate": 7.141952118318961e-09,
+ "loss": 0.113973468542099,
+ "num_tokens": 1437333.0,
+ "reward": 0.5760529637336731,
+ "reward_std": 0.11078595370054245,
+ "rewards/true_env_reward_fn/mean": 0.5760529637336731,
+ "rewards/true_env_reward_fn/std": 0.11078596115112305,
+ "step": 350,
+ "step_time": 3.624226386000373
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 1437333,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-350/training_args.bin b/checkpoint-350/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-350/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-369/README.md b/checkpoint-369/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d43f5cce16542812c28205685858b78648c39bdb
--- /dev/null
+++ b/checkpoint-369/README.md
@@ -0,0 +1,209 @@
+---
+base_model: unsloth/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-369/adapter_config.json b/checkpoint-369/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-369/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-369/adapter_model.safetensors b/checkpoint-369/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..91858393ce9c2d5cb7d204530d11113af62ab24d
--- /dev/null
+++ b/checkpoint-369/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d50e9249dbc53f60027557052a860b27022847dd92972fcb41cbf4f17869a3
+size 8731128
diff --git a/checkpoint-369/chat_template.jinja b/checkpoint-369/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/checkpoint-369/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/checkpoint-369/optimizer.pt b/checkpoint-369/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..244da5c2da4bb2c88b492e1ff8fc15b4f0aa74e2
--- /dev/null
+++ b/checkpoint-369/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b59c45a302c3ae6982a84823f8a53bbea848538c34b82dbfa2fe549fbebf96a
+size 17526842
diff --git a/checkpoint-369/ref/adapter_config.json b/checkpoint-369/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-369/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-369/ref/adapter_model.safetensors b/checkpoint-369/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-369/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-369/rng_state.pth b/checkpoint-369/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cf9102c488bf97243fb26169f0d2a5bf7b4fa574
--- /dev/null
+++ b/checkpoint-369/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cddf9809a20728eb66ae92c787b9f197e770ea3791307df20b8415995cf8cd5
+size 14244
diff --git a/checkpoint-369/scaler.pt b/checkpoint-369/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d29e45120aeb701c0af469284f2b69b48a73338d
--- /dev/null
+++ b/checkpoint-369/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:223422cfc1cf170d3568be44ebcfa2a75193968b657d45c3e4a6a5ba95df4bed
+size 988
diff --git a/checkpoint-369/scheduler.pt b/checkpoint-369/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed55e66f59b4adac8f2ae667f838fb75a4c89e39
--- /dev/null
+++ b/checkpoint-369/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e29bd521096c56db781188e21860224ef702e4b2bf01077bbe5e9dfdbfcbf96d
+size 1064
diff --git a/checkpoint-369/tokenizer.json b/checkpoint-369/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c
--- /dev/null
+++ b/checkpoint-369/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-369/tokenizer_config.json b/checkpoint-369/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9aa5955b7dd3b15995a880b534c69ad538218d2d
--- /dev/null
+++ b/checkpoint-369/tokenizer_config.json
@@ -0,0 +1,16 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "is_local": true,
+ "local_files_only": false,
+ "model_max_length": 32768,
+ "pad_token": "<|im_end|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/checkpoint-369/trainer_state.json b/checkpoint-369/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1cf8e9265c1dd7c178edb618ba81fb9ffd5034ee
--- /dev/null
+++ b/checkpoint-369/trainer_state.json
@@ -0,0 +1,10366 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 500,
+ "global_step": 369,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
+ "learning_rate": 0.0,
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 0.9613964557647705,
+ "epoch": 0.4146341463414634,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12812700867652893,
+ "kl": 1.0464088063599775e-05,
+ "learning_rate": 7.969773173984153e-07,
+ "loss": 0.023206032812595367,
+ "num_tokens": 210443.0,
+ "reward": 0.3208000063896179,
+ "reward_std": 0.25050169229507446,
+ "rewards/true_env_reward_fn/mean": 0.3208000063896179,
+ "rewards/true_env_reward_fn/std": 0.25050172209739685,
+ "step": 51,
+ "step_time": 3.6275602460009395
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2359730005264282,
+ "epoch": 0.42276422764227645,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1384950578212738,
+ "kl": 1.2094554222130682e-05,
+ "learning_rate": 7.964951099967749e-07,
+ "loss": -0.07054222375154495,
+ "num_tokens": 213833.0,
+ "reward": 0.5900156497955322,
+ "reward_std": 0.18237514793872833,
+ "rewards/true_env_reward_fn/mean": 0.5900156497955322,
+ "rewards/true_env_reward_fn/std": 0.18237514793872833,
+ "step": 52,
+ "step_time": 3.8849526029989647
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 79.5,
+ "completions/mean_terminated_length": 79.5,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2706108689308167,
+ "epoch": 0.43089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17388403415679932,
+ "kl": 1.3583369309344562e-05,
+ "learning_rate": 7.959774001575264e-07,
+ "loss": 0.06114684417843819,
+ "num_tokens": 216853.0,
+ "reward": 0.4848448634147644,
+ "reward_std": 0.2859330177307129,
+ "rewards/true_env_reward_fn/mean": 0.4848448634147644,
+ "rewards/true_env_reward_fn/std": 0.2859330177307129,
+ "step": 53,
+ "step_time": 4.964324356000361
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2430712580680847,
+ "epoch": 0.43902439024390244,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11113106459379196,
+ "kl": 1.2204414360894589e-05,
+ "learning_rate": 7.954242342367553e-07,
+ "loss": 0.010590985417366028,
+ "num_tokens": 221252.0,
+ "reward": 0.392258882522583,
+ "reward_std": 0.13280020654201508,
+ "rewards/true_env_reward_fn/mean": 0.392258882522583,
+ "rewards/true_env_reward_fn/std": 0.13280019164085388,
+ "step": 54,
+ "step_time": 3.5511989209990134
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 118.0,
+ "completions/max_terminated_length": 118.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3175880908966064,
+ "epoch": 0.44715447154471544,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20881597697734833,
+ "kl": 1.58558846123924e-05,
+ "learning_rate": 7.948356617653087e-07,
+ "loss": -0.06772151589393616,
+ "num_tokens": 224691.0,
+ "reward": 0.30961817502975464,
+ "reward_std": 0.27422165870666504,
+ "rewards/true_env_reward_fn/mean": 0.30961817502975464,
+ "rewards/true_env_reward_fn/std": 0.27422165870666504,
+ "step": 55,
+ "step_time": 5.031640098000935
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 64.625,
+ "completions/mean_terminated_length": 64.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.4056915640830994,
+ "epoch": 0.45528455284552843,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.606108895037323e-05,
+ "kl": 1.2847603557020193e-05,
+ "learning_rate": 7.942117354443597e-07,
+ "loss": 6.408997705875663e-07,
+ "num_tokens": 228116.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 56,
+ "step_time": 3.6221305880008003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.4034882187843323,
+ "epoch": 0.4634146341463415,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19157616794109344,
+ "kl": 1.4551038475474343e-05,
+ "learning_rate": 7.935525111406885e-07,
+ "loss": 0.021202675998210907,
+ "num_tokens": 233139.0,
+ "reward": 0.32785865664482117,
+ "reward_std": 0.2835054397583008,
+ "rewards/true_env_reward_fn/mean": 0.32785865664482117,
+ "rewards/true_env_reward_fn/std": 0.28350546956062317,
+ "step": 57,
+ "step_time": 3.7005361410010664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 173.0,
+ "completions/max_terminated_length": 173.0,
+ "completions/mean_length": 76.625,
+ "completions/mean_terminated_length": 76.625,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2941595911979675,
+ "epoch": 0.4715447154471545,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14299193024635315,
+ "kl": 1.3164159554435173e-05,
+ "learning_rate": 7.92858047881681e-07,
+ "loss": -0.14726585149765015,
+ "num_tokens": 238584.0,
+ "reward": 0.444433331489563,
+ "reward_std": 0.030650291591882706,
+ "rewards/true_env_reward_fn/mean": 0.444433331489563,
+ "rewards/true_env_reward_fn/std": 0.030650287866592407,
+ "step": 58,
+ "step_time": 7.550715425000817
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1917714476585388,
+ "epoch": 0.4796747967479675,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25083038210868835,
+ "kl": 1.3176229913369752e-05,
+ "learning_rate": 7.921284078500422e-07,
+ "loss": 0.088463693857193,
+ "num_tokens": 240669.0,
+ "reward": 0.7982887029647827,
+ "reward_std": 0.1672983169555664,
+ "rewards/true_env_reward_fn/mean": 0.7982887029647827,
+ "rewards/true_env_reward_fn/std": 0.1672983169555664,
+ "step": 59,
+ "step_time": 3.7769912429994292
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.375,
+ "completions/mean_terminated_length": 66.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3743653893470764,
+ "epoch": 0.4878048780487805,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18919643759727478,
+ "kl": 1.231462101713987e-05,
+ "learning_rate": 7.91363656378229e-07,
+ "loss": -0.08548973500728607,
+ "num_tokens": 243808.0,
+ "reward": 0.5988538861274719,
+ "reward_std": 0.11870570480823517,
+ "rewards/true_env_reward_fn/mean": 0.5988538861274719,
+ "rewards/true_env_reward_fn/std": 0.11870571970939636,
+ "step": 60,
+ "step_time": 4.052767743998629
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 77.125,
+ "completions/mean_terminated_length": 77.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.329764723777771,
+ "epoch": 0.4959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1690233051776886,
+ "kl": 1.405783814334427e-05,
+ "learning_rate": 7.905638619426003e-07,
+ "loss": 0.0050433604046702385,
+ "num_tokens": 248725.0,
+ "reward": 0.27516257762908936,
+ "reward_std": 0.32322537899017334,
+ "rewards/true_env_reward_fn/mean": 0.27516257762908936,
+ "rewards/true_env_reward_fn/std": 0.32322537899017334,
+ "step": 61,
+ "step_time": 6.010593229999358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.2542970776557922,
+ "epoch": 0.5040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11462891101837158,
+ "kl": 1.13775058707688e-05,
+ "learning_rate": 7.897290961572853e-07,
+ "loss": -0.007184989750385284,
+ "num_tokens": 252101.0,
+ "reward": 0.5372380018234253,
+ "reward_std": 0.13500821590423584,
+ "rewards/true_env_reward_fn/mean": 0.5372380018234253,
+ "rewards/true_env_reward_fn/std": 0.13500821590423584,
+ "step": 62,
+ "step_time": 3.4512634010006877
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 65.75,
+ "completions/mean_terminated_length": 65.75,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1982964873313904,
+ "epoch": 0.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12309258431196213,
+ "kl": 1.69004347299051e-05,
+ "learning_rate": 7.888594337677712e-07,
+ "loss": 0.0009508281946182251,
+ "num_tokens": 255231.0,
+ "reward": 0.6114543080329895,
+ "reward_std": 0.10413603484630585,
+ "rewards/true_env_reward_fn/mean": 0.6114543080329895,
+ "rewards/true_env_reward_fn/std": 0.10413603484630585,
+ "step": 63,
+ "step_time": 3.735559521997857
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 69.0,
+ "completions/mean_terminated_length": 69.0,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3587612509727478,
+ "epoch": 0.5203252032520326,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15858450531959534,
+ "kl": 1.4598341294913553e-05,
+ "learning_rate": 7.879549526442108e-07,
+ "loss": 0.0696716383099556,
+ "num_tokens": 260523.0,
+ "reward": 0.2912999987602234,
+ "reward_std": 0.2844822406768799,
+ "rewards/true_env_reward_fn/mean": 0.2912999987602234,
+ "rewards/true_env_reward_fn/std": 0.2844822406768799,
+ "step": 64,
+ "step_time": 5.731267729999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 230.0,
+ "completions/max_terminated_length": 230.0,
+ "completions/mean_length": 83.5,
+ "completions/mean_terminated_length": 83.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2284430861473083,
+ "epoch": 0.5284552845528455,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13374178111553192,
+ "kl": 1.2341822639427846e-05,
+ "learning_rate": 7.870157337744494e-07,
+ "loss": 0.10693901032209396,
+ "num_tokens": 264967.0,
+ "reward": 0.3284733295440674,
+ "reward_std": 0.3848404288291931,
+ "rewards/true_env_reward_fn/mean": 0.3284733295440674,
+ "rewards/true_env_reward_fn/std": 0.3848404288291931,
+ "step": 65,
+ "step_time": 9.601442954000959
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2396279573440552,
+ "epoch": 0.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08427461981773376,
+ "kl": 1.4658115105703473e-05,
+ "learning_rate": 7.860418612567733e-07,
+ "loss": -0.05642998591065407,
+ "num_tokens": 269717.0,
+ "reward": 0.38946664333343506,
+ "reward_std": 0.1897086799144745,
+ "rewards/true_env_reward_fn/mean": 0.38946664333343506,
+ "rewards/true_env_reward_fn/std": 0.1897086799144745,
+ "step": 66,
+ "step_time": 6.017849919000582
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.2245049476623535,
+ "epoch": 0.5447154471544715,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13357418775558472,
+ "kl": 1.2806529412046075e-05,
+ "learning_rate": 7.850334222923798e-07,
+ "loss": 0.03744228184223175,
+ "num_tokens": 275407.0,
+ "reward": 0.08966667205095291,
+ "reward_std": 0.23612774908542633,
+ "rewards/true_env_reward_fn/mean": 0.08966667205095291,
+ "rewards/true_env_reward_fn/std": 0.23612776398658752,
+ "step": 67,
+ "step_time": 4.4363536659984675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2679882645606995,
+ "epoch": 0.5528455284552846,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15667268633842468,
+ "kl": 1.2213955869810889e-05,
+ "learning_rate": 7.83990507177569e-07,
+ "loss": -0.052396662533283234,
+ "num_tokens": 280838.0,
+ "reward": 0.2431039959192276,
+ "reward_std": 0.2672288715839386,
+ "rewards/true_env_reward_fn/mean": 0.2431039959192276,
+ "rewards/true_env_reward_fn/std": 0.2672288715839386,
+ "step": 68,
+ "step_time": 3.6370441849994677
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 62.625,
+ "completions/mean_terminated_length": 62.625,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2563416361808777,
+ "epoch": 0.5609756097560976,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.937557868193835e-05,
+ "kl": 1.1138304216729011e-05,
+ "learning_rate": 7.829132092956586e-07,
+ "loss": 5.569941095018294e-07,
+ "num_tokens": 283603.0,
+ "reward": 0.6040733456611633,
+ "reward_std": 0.0834638923406601,
+ "rewards/true_env_reward_fn/mean": 0.6040733456611633,
+ "rewards/true_env_reward_fn/std": 0.08346389979124069,
+ "step": 69,
+ "step_time": 3.466609713001162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 126.0,
+ "completions/mean_terminated_length": 126.0,
+ "completions/min_length": 76.0,
+ "completions/min_terminated_length": 76.0,
+ "entropy": 1.8668264746665955,
+ "epoch": 0.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11100972443819046,
+ "kl": 1.3833193406753708e-05,
+ "learning_rate": 7.81801625108622e-07,
+ "loss": -0.04258224368095398,
+ "num_tokens": 290511.0,
+ "reward": 0.37345871329307556,
+ "reward_std": 0.016035744920372963,
+ "rewards/true_env_reward_fn/mean": 0.37345871329307556,
+ "rewards/true_env_reward_fn/std": 0.016035741195082664,
+ "step": 70,
+ "step_time": 8.357124549000218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.1647167801856995,
+ "epoch": 0.5772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12842044234275818,
+ "kl": 1.35402724481537e-05,
+ "learning_rate": 7.806558541484517e-07,
+ "loss": -0.0010651163756847382,
+ "num_tokens": 294315.0,
+ "reward": 0.6432806849479675,
+ "reward_std": 0.2300010770559311,
+ "rewards/true_env_reward_fn/mean": 0.6432806849479675,
+ "rewards/true_env_reward_fn/std": 0.23000109195709229,
+ "step": 71,
+ "step_time": 3.8402047919989855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1465299725532532,
+ "epoch": 0.5853658536585366,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23560228943824768,
+ "kl": 1.4576367902918719e-05,
+ "learning_rate": 7.794759990082466e-07,
+ "loss": -0.11232151836156845,
+ "num_tokens": 297803.0,
+ "reward": 0.30700522661209106,
+ "reward_std": 0.3690750300884247,
+ "rewards/true_env_reward_fn/mean": 0.30700522661209106,
+ "rewards/true_env_reward_fn/std": 0.3690750002861023,
+ "step": 72,
+ "step_time": 3.467162693001228
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.2479569911956787,
+ "epoch": 0.5934959349593496,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011426099081290886,
+ "kl": 1.304310217165039e-05,
+ "learning_rate": 7.782621653330256e-07,
+ "loss": 6.391838383024151e-07,
+ "num_tokens": 301427.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 73,
+ "step_time": 5.824168748999
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.170280933380127,
+ "epoch": 0.6016260162601627,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22593456506729126,
+ "kl": 2.0052431864314713e-05,
+ "learning_rate": 7.77014461810269e-07,
+ "loss": 0.16111303865909576,
+ "num_tokens": 305492.0,
+ "reward": 0.3909183144569397,
+ "reward_std": 0.21756574511528015,
+ "rewards/true_env_reward_fn/mean": 0.3909183144569397,
+ "rewards/true_env_reward_fn/std": 0.21756574511528015,
+ "step": 74,
+ "step_time": 4.510902927002462
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2373355031013489,
+ "epoch": 0.6097560975609756,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.604005466215312e-05,
+ "kl": 1.0138399375136942e-05,
+ "learning_rate": 7.757330001601855e-07,
+ "loss": 5.069200028628984e-07,
+ "num_tokens": 309826.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 75,
+ "step_time": 3.6695911980004894
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.1873346865177155,
+ "epoch": 0.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2067025899887085,
+ "kl": 1.4842833934380906e-05,
+ "learning_rate": 7.744178951257091e-07,
+ "loss": -0.036428727209568024,
+ "num_tokens": 316885.0,
+ "reward": 0.13499999046325684,
+ "reward_std": 0.23260429501533508,
+ "rewards/true_env_reward_fn/mean": 0.13499999046325684,
+ "rewards/true_env_reward_fn/std": 0.23260430991649628,
+ "step": 76,
+ "step_time": 4.359561059001862
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0709484219551086,
+ "epoch": 0.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18151503801345825,
+ "kl": 1.3910183042753488e-05,
+ "learning_rate": 7.730692644622251e-07,
+ "loss": -0.06179043650627136,
+ "num_tokens": 319230.0,
+ "reward": 0.6732838153839111,
+ "reward_std": 0.1450435221195221,
+ "rewards/true_env_reward_fn/mean": 0.6732838153839111,
+ "rewards/true_env_reward_fn/std": 0.14504355192184448,
+ "step": 77,
+ "step_time": 3.1786108079995756
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 107.0,
+ "completions/max_terminated_length": 107.0,
+ "completions/mean_length": 72.375,
+ "completions/mean_terminated_length": 72.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5439093112945557,
+ "epoch": 0.6341463414634146,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20670665800571442,
+ "kl": 1.7317805031780154e-05,
+ "learning_rate": 7.716872289270261e-07,
+ "loss": -0.0654018223285675,
+ "num_tokens": 324633.0,
+ "reward": 0.23838475346565247,
+ "reward_std": 0.2594907879829407,
+ "rewards/true_env_reward_fn/mean": 0.23838475346565247,
+ "rewards/true_env_reward_fn/std": 0.2594907879829407,
+ "step": 78,
+ "step_time": 4.930556027000421
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 47.625,
+ "completions/mean_terminated_length": 47.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1996066868305206,
+ "epoch": 0.6422764227642277,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21137002110481262,
+ "kl": 1.325221819570288e-05,
+ "learning_rate": 7.702719122684991e-07,
+ "loss": 0.003889208659529686,
+ "num_tokens": 329142.0,
+ "reward": 0.3934500217437744,
+ "reward_std": 0.1389254629611969,
+ "rewards/true_env_reward_fn/mean": 0.3934500217437744,
+ "rewards/true_env_reward_fn/std": 0.1389254778623581,
+ "step": 79,
+ "step_time": 3.5688320999997813
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4094278812408447,
+ "epoch": 0.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17559278011322021,
+ "kl": 1.6261046312138205e-05,
+ "learning_rate": 7.688234412150453e-07,
+ "loss": -0.04887707903981209,
+ "num_tokens": 331663.0,
+ "reward": 0.49859046936035156,
+ "reward_std": 0.12171231955289841,
+ "rewards/true_env_reward_fn/mean": 0.49859046936035156,
+ "rewards/true_env_reward_fn/std": 0.12171231955289841,
+ "step": 80,
+ "step_time": 3.7867210379990865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1693094372749329,
+ "epoch": 0.6585365853658537,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010281114373356104,
+ "kl": 1.2930718639836414e-05,
+ "learning_rate": 7.673419454637328e-07,
+ "loss": 6.465359092544531e-07,
+ "num_tokens": 334637.0,
+ "reward": 0.5707399845123291,
+ "reward_std": 0.11909874528646469,
+ "rewards/true_env_reward_fn/mean": 0.5707399845123291,
+ "rewards/true_env_reward_fn/std": 0.1190987378358841,
+ "step": 81,
+ "step_time": 3.4751437539998733
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3214005827903748,
+ "epoch": 0.6666666666666666,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2361973226070404,
+ "kl": 1.4227861356630456e-05,
+ "learning_rate": 7.658275576686829e-07,
+ "loss": -0.08402466773986816,
+ "num_tokens": 341701.0,
+ "reward": 0.09331665933132172,
+ "reward_std": 0.2172754853963852,
+ "rewards/true_env_reward_fn/mean": 0.09331665933132172,
+ "rewards/true_env_reward_fn/std": 0.2172755002975464,
+ "step": 82,
+ "step_time": 4.433740980000948
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 142.375,
+ "completions/mean_terminated_length": 89.5714340209961,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.817092776298523,
+ "epoch": 0.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.11815346032381058,
+ "kl": 1.6899173715501092e-05,
+ "learning_rate": 7.642804134291927e-07,
+ "loss": -0.09939523041248322,
+ "num_tokens": 346380.0,
+ "reward": 0.47429025173187256,
+ "reward_std": 0.24831563234329224,
+ "rewards/true_env_reward_fn/mean": 0.47429025173187256,
+ "rewards/true_env_reward_fn/std": 0.24831561744213104,
+ "step": 83,
+ "step_time": 20.738665008999305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2211430668830872,
+ "epoch": 0.6829268292682927,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20952872931957245,
+ "kl": 1.2894654446427012e-05,
+ "learning_rate": 7.62700651277593e-07,
+ "loss": -0.0016747117042541504,
+ "num_tokens": 351186.0,
+ "reward": 0.386501669883728,
+ "reward_std": 0.17392057180404663,
+ "rewards/true_env_reward_fn/mean": 0.386501669883728,
+ "rewards/true_env_reward_fn/std": 0.17392057180404663,
+ "step": 84,
+ "step_time": 4.028964023000299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4367225170135498,
+ "epoch": 0.6910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18939745426177979,
+ "kl": 1.6035403859859798e-05,
+ "learning_rate": 7.610884126668449e-07,
+ "loss": 0.0628451332449913,
+ "num_tokens": 355999.0,
+ "reward": 0.5092726349830627,
+ "reward_std": 0.2734805643558502,
+ "rewards/true_env_reward_fn/mean": 0.5092726349830627,
+ "rewards/true_env_reward_fn/std": 0.2734805941581726,
+ "step": 85,
+ "step_time": 4.244558566999331
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.25,
+ "completions/mean_terminated_length": 44.25,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.009476900100708,
+ "epoch": 0.6991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22703228890895844,
+ "kl": 1.2845549463236239e-05,
+ "learning_rate": 7.594438419578729e-07,
+ "loss": -0.005728684365749359,
+ "num_tokens": 360925.0,
+ "reward": 0.28028765320777893,
+ "reward_std": 0.2404259443283081,
+ "rewards/true_env_reward_fn/mean": 0.28028765320777893,
+ "rewards/true_env_reward_fn/std": 0.2404259443283081,
+ "step": 86,
+ "step_time": 2.618181756000922
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.1686812043190002,
+ "epoch": 0.7073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.401248098583892e-05,
+ "kl": 1.2304412848607171e-05,
+ "learning_rate": 7.577670864066391e-07,
+ "loss": 6.143833388705389e-07,
+ "num_tokens": 362399.0,
+ "reward": 0.768503725528717,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.768503725528717,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 87,
+ "step_time": 3.34067542199773
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0876938998699188,
+ "epoch": 0.7154471544715447,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010135328921023756,
+ "kl": 1.3493038295564475e-05,
+ "learning_rate": 7.560582961509586e-07,
+ "loss": 6.750068450855906e-07,
+ "num_tokens": 365500.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 88,
+ "step_time": 3.3087227100004384
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.0288619995117188,
+ "epoch": 0.7235772357723578,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010261479474138469,
+ "kl": 1.3740621852775803e-05,
+ "learning_rate": 7.543176241970547e-07,
+ "loss": 6.875395683891838e-07,
+ "num_tokens": 369222.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 89,
+ "step_time": 3.786183243999403
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1757304668426514,
+ "epoch": 0.7317073170731707,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2307090163230896,
+ "kl": 2.1445125639729667e-05,
+ "learning_rate": 7.525452264058595e-07,
+ "loss": 0.12042637169361115,
+ "num_tokens": 373465.0,
+ "reward": 0.4571714401245117,
+ "reward_std": 0.39374110102653503,
+ "rewards/true_env_reward_fn/mean": 0.4571714401245117,
+ "rewards/true_env_reward_fn/std": 0.39374107122421265,
+ "step": 90,
+ "step_time": 3.9787140030002774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.302090346813202,
+ "epoch": 0.7398373983739838,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16624286770820618,
+ "kl": 1.6463789506815374e-05,
+ "learning_rate": 7.507412614790579e-07,
+ "loss": -0.05975423753261566,
+ "num_tokens": 378029.0,
+ "reward": 0.3388232886791229,
+ "reward_std": 0.2467346489429474,
+ "rewards/true_env_reward_fn/mean": 0.3388232886791229,
+ "rewards/true_env_reward_fn/std": 0.24673466384410858,
+ "step": 91,
+ "step_time": 3.9565000490001694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 77.75,
+ "completions/mean_terminated_length": 77.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2768036723136902,
+ "epoch": 0.7479674796747967,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10557293146848679,
+ "kl": 1.2602345123013947e-05,
+ "learning_rate": 7.489058909448776e-07,
+ "loss": -0.023296140134334564,
+ "num_tokens": 380883.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 92,
+ "step_time": 4.720347813999979
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2670618891716003,
+ "epoch": 0.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14662617444992065,
+ "kl": 1.812677373891347e-05,
+ "learning_rate": 7.470392791436244e-07,
+ "loss": -0.05785401538014412,
+ "num_tokens": 386095.0,
+ "reward": 0.30487915873527527,
+ "reward_std": 0.24597851932048798,
+ "rewards/true_env_reward_fn/mean": 0.30487915873527527,
+ "rewards/true_env_reward_fn/std": 0.24597853422164917,
+ "step": 93,
+ "step_time": 3.1318131530006212
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1208478510379791,
+ "epoch": 0.7642276422764228,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011506211740197614,
+ "kl": 1.2571507795655634e-05,
+ "learning_rate": 7.451415932129691e-07,
+ "loss": 6.294373520177032e-07,
+ "num_tokens": 388335.0,
+ "reward": 0.7244763970375061,
+ "reward_std": 0.23028412461280823,
+ "rewards/true_env_reward_fn/mean": 0.7244763970375061,
+ "rewards/true_env_reward_fn/std": 0.23028412461280823,
+ "step": 94,
+ "step_time": 3.6959203189999243
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1998488903045654,
+ "epoch": 0.7723577235772358,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12171207368373871,
+ "kl": 1.6534771020815242e-05,
+ "learning_rate": 7.432130030729804e-07,
+ "loss": 0.05708106979727745,
+ "num_tokens": 393029.0,
+ "reward": 0.29566600918769836,
+ "reward_std": 0.2818882167339325,
+ "rewards/true_env_reward_fn/mean": 0.29566600918769836,
+ "rewards/true_env_reward_fn/std": 0.2818882167339325,
+ "step": 95,
+ "step_time": 4.322851452001487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0649794340133667,
+ "epoch": 0.7804878048780488,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2012680470943451,
+ "kl": 1.1399301456549438e-05,
+ "learning_rate": 7.412536814109106e-07,
+ "loss": -0.05478152632713318,
+ "num_tokens": 398112.0,
+ "reward": 0.23480799794197083,
+ "reward_std": 0.28209570050239563,
+ "rewards/true_env_reward_fn/mean": 0.23480799794197083,
+ "rewards/true_env_reward_fn/std": 0.282095730304718,
+ "step": 96,
+ "step_time": 3.4046103930013487
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.5,
+ "completions/mean_terminated_length": 56.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.4298859238624573,
+ "epoch": 0.7886178861788617,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2239074409008026,
+ "kl": 3.293174540885957e-05,
+ "learning_rate": 7.392638036657332e-07,
+ "loss": 0.09779056906700134,
+ "num_tokens": 402892.0,
+ "reward": 0.13796034455299377,
+ "reward_std": 0.22141560912132263,
+ "rewards/true_env_reward_fn/mean": 0.13796034455299377,
+ "rewards/true_env_reward_fn/std": 0.22141562402248383,
+ "step": 97,
+ "step_time": 3.779275342998517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 52.0,
+ "completions/mean_terminated_length": 52.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2070425152778625,
+ "epoch": 0.7967479674796748,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.19742031395435333,
+ "kl": 1.4374184502230491e-05,
+ "learning_rate": 7.372435480124337e-07,
+ "loss": -0.006231316365301609,
+ "num_tokens": 408052.0,
+ "reward": 0.43320000171661377,
+ "reward_std": 0.05237230286002159,
+ "rewards/true_env_reward_fn/mean": 0.43320000171661377,
+ "rewards/true_env_reward_fn/std": 0.052372295409440994,
+ "step": 98,
+ "step_time": 3.1304682769987267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 0.9987849593162537,
+ "epoch": 0.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.897383668227121e-05,
+ "kl": 1.2614300885616103e-05,
+ "learning_rate": 7.35193095346056e-07,
+ "loss": 6.314263600870618e-07,
+ "num_tokens": 409605.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 99,
+ "step_time": 4.13536422299876
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.0226224660873413,
+ "epoch": 0.8130081300813008,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20178858935832977,
+ "kl": 1.0500047665118473e-05,
+ "learning_rate": 7.331126292655044e-07,
+ "loss": -0.17970919609069824,
+ "num_tokens": 411488.0,
+ "reward": 0.6963247060775757,
+ "reward_std": 0.18840119242668152,
+ "rewards/true_env_reward_fn/mean": 0.6963247060775757,
+ "rewards/true_env_reward_fn/std": 0.1884012222290039,
+ "step": 100,
+ "step_time": 3.7544156769981782
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.0,
+ "completions/mean_terminated_length": 59.0,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2509461045265198,
+ "epoch": 0.8211382113821138,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22887632250785828,
+ "kl": 2.1612477212329395e-05,
+ "learning_rate": 7.310023360571047e-07,
+ "loss": 0.025605827569961548,
+ "num_tokens": 414080.0,
+ "reward": 0.588032603263855,
+ "reward_std": 0.11032751202583313,
+ "rewards/true_env_reward_fn/mean": 0.588032603263855,
+ "rewards/true_env_reward_fn/std": 0.11032749712467194,
+ "step": 101,
+ "step_time": 3.625197022998691
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 176.0,
+ "completions/max_terminated_length": 176.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5443179607391357,
+ "epoch": 0.8292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10614532232284546,
+ "kl": 1.492139062975184e-05,
+ "learning_rate": 7.28862404677924e-07,
+ "loss": 0.06531564146280289,
+ "num_tokens": 419835.0,
+ "reward": 0.07074306160211563,
+ "reward_std": 0.2918013632297516,
+ "rewards/true_env_reward_fn/mean": 0.07074306160211563,
+ "rewards/true_env_reward_fn/std": 0.2918013632297516,
+ "step": 102,
+ "step_time": 7.796810614998321
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 101.0,
+ "completions/max_terminated_length": 101.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3223788738250732,
+ "epoch": 0.8373983739837398,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2421368807554245,
+ "kl": 3.265505938543356e-05,
+ "learning_rate": 7.266930267388503e-07,
+ "loss": -0.07752113044261932,
+ "num_tokens": 422773.0,
+ "reward": 0.33568501472473145,
+ "reward_std": 0.2780380845069885,
+ "rewards/true_env_reward_fn/mean": 0.33568501472473145,
+ "rewards/true_env_reward_fn/std": 0.2780380845069885,
+ "step": 103,
+ "step_time": 4.313938073000827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 67.25,
+ "completions/mean_terminated_length": 67.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3332037329673767,
+ "epoch": 0.8455284552845529,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13883370161056519,
+ "kl": 2.1224042484391248e-05,
+ "learning_rate": 7.244943964874369e-07,
+ "loss": 0.021739646792411804,
+ "num_tokens": 426507.0,
+ "reward": 0.40595096349716187,
+ "reward_std": 0.2035457342863083,
+ "rewards/true_env_reward_fn/mean": 0.40595096349716187,
+ "rewards/true_env_reward_fn/std": 0.2035457193851471,
+ "step": 104,
+ "step_time": 4.155937195999286
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3391229510307312,
+ "epoch": 0.8536585365853658,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24205009639263153,
+ "kl": 2.5022183763212524e-05,
+ "learning_rate": 7.222667107905085e-07,
+ "loss": 0.06330433487892151,
+ "num_tokens": 429010.0,
+ "reward": 0.3355163037776947,
+ "reward_std": 0.2902730703353882,
+ "rewards/true_env_reward_fn/mean": 0.3355163037776947,
+ "rewards/true_env_reward_fn/std": 0.29027310013771057,
+ "step": 105,
+ "step_time": 3.808478789000219
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1864720582962036,
+ "epoch": 0.8617886178861789,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13473568856716156,
+ "kl": 1.4212585938366828e-05,
+ "learning_rate": 7.200101691165338e-07,
+ "loss": -0.020715661346912384,
+ "num_tokens": 432403.0,
+ "reward": 0.4871198534965515,
+ "reward_std": 0.15407639741897583,
+ "rewards/true_env_reward_fn/mean": 0.4871198534965515,
+ "rewards/true_env_reward_fn/std": 0.15407641232013702,
+ "step": 106,
+ "step_time": 4.240638332001254
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.0,
+ "completions/mean_terminated_length": 48.0,
+ "completions/min_length": 24.0,
+ "completions/min_terminated_length": 24.0,
+ "entropy": 1.0669284462928772,
+ "epoch": 0.8699186991869918,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14131899178028107,
+ "kl": 1.5787159554747632e-05,
+ "learning_rate": 7.177249735177651e-07,
+ "loss": 0.03678784519433975,
+ "num_tokens": 435995.0,
+ "reward": 0.5010770559310913,
+ "reward_std": 0.48966261744499207,
+ "rewards/true_env_reward_fn/mean": 0.5010770559310913,
+ "rewards/true_env_reward_fn/std": 0.48966261744499207,
+ "step": 107,
+ "step_time": 3.3587191269998584
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 71.875,
+ "completions/mean_terminated_length": 71.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.304731547832489,
+ "epoch": 0.8780487804878049,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.380985673284158e-05,
+ "kl": 1.3128728824085556e-05,
+ "learning_rate": 7.154113286121462e-07,
+ "loss": 6.494262834166875e-07,
+ "num_tokens": 442094.0,
+ "reward": 0.4055500030517578,
+ "reward_std": 0.052258480340242386,
+ "rewards/true_env_reward_fn/mean": 0.4055500030517578,
+ "rewards/true_env_reward_fn/std": 0.052258484065532684,
+ "step": 108,
+ "step_time": 4.337008413998774
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2324069738388062,
+ "epoch": 0.8861788617886179,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13588950037956238,
+ "kl": 1.3448377558233915e-05,
+ "learning_rate": 7.130694415649912e-07,
+ "loss": 1.0952353477478027e-06,
+ "num_tokens": 447226.0,
+ "reward": 0.20854972302913666,
+ "reward_std": 0.06059705466032028,
+ "rewards/true_env_reward_fn/mean": 0.20854972302913666,
+ "rewards/true_env_reward_fn/std": 0.06059705838561058,
+ "step": 109,
+ "step_time": 3.2976038649994734
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1567262411117554,
+ "epoch": 0.8943089430894309,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.736967720324174e-05,
+ "kl": 1.2838129805459175e-05,
+ "learning_rate": 7.106995220704342e-07,
+ "loss": 6.425898391171359e-07,
+ "num_tokens": 450359.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 110,
+ "step_time": 4.067084037998939
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 65.0,
+ "completions/mean_terminated_length": 65.0,
+ "completions/min_length": 12.0,
+ "completions/min_terminated_length": 12.0,
+ "entropy": 1.496058464050293,
+ "epoch": 0.9024390243902439,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18041981756687164,
+ "kl": 1.6616825632809196e-05,
+ "learning_rate": 7.083017823326532e-07,
+ "loss": 0.0269068144261837,
+ "num_tokens": 453583.0,
+ "reward": 0.5647265911102295,
+ "reward_std": 0.1507105529308319,
+ "rewards/true_env_reward_fn/mean": 0.5647265911102295,
+ "rewards/true_env_reward_fn/std": 0.1507105529308319,
+ "step": 111,
+ "step_time": 4.347732382997492
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 67.75,
+ "completions/mean_terminated_length": 67.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.4196155667304993,
+ "epoch": 0.9105691056910569,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18451112508773804,
+ "kl": 2.1803500203532167e-05,
+ "learning_rate": 7.058764370468698e-07,
+ "loss": 0.1650262475013733,
+ "num_tokens": 456773.0,
+ "reward": 0.6907394528388977,
+ "reward_std": 0.1393815129995346,
+ "rewards/true_env_reward_fn/mean": 0.6907394528388977,
+ "rewards/true_env_reward_fn/std": 0.1393815129995346,
+ "step": 112,
+ "step_time": 4.627644968999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2243221998214722,
+ "epoch": 0.9186991869918699,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331371545791626,
+ "kl": 1.548633599668392e-05,
+ "learning_rate": 7.034237033801247e-07,
+ "loss": 0.039844345301389694,
+ "num_tokens": 462714.0,
+ "reward": 0.21676866710186005,
+ "reward_std": 0.26559779047966003,
+ "rewards/true_env_reward_fn/mean": 0.21676866710186005,
+ "rewards/true_env_reward_fn/std": 0.26559779047966003,
+ "step": 113,
+ "step_time": 3.8455466220002563
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 73.0,
+ "completions/mean_terminated_length": 73.0,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.325823724269867,
+ "epoch": 0.926829268292683,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1866220384836197,
+ "kl": 1.8801019905367866e-05,
+ "learning_rate": 7.009438009518325e-07,
+ "loss": 0.06504581868648529,
+ "num_tokens": 465994.0,
+ "reward": 0.5194582939147949,
+ "reward_std": 0.2796703577041626,
+ "rewards/true_env_reward_fn/mean": 0.5194582939147949,
+ "rewards/true_env_reward_fn/std": 0.2796703577041626,
+ "step": 114,
+ "step_time": 4.151028698999653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1521879434585571,
+ "epoch": 0.9349593495934959,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14320029318332672,
+ "kl": 1.2749982033710694e-05,
+ "learning_rate": 6.98436951814117e-07,
+ "loss": 0.03685300797224045,
+ "num_tokens": 468615.0,
+ "reward": 0.5399107933044434,
+ "reward_std": 0.26432597637176514,
+ "rewards/true_env_reward_fn/mean": 0.5399107933044434,
+ "rewards/true_env_reward_fn/std": 0.26432597637176514,
+ "step": 115,
+ "step_time": 3.6973990600017714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.201507806777954,
+ "epoch": 0.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14264807105064392,
+ "kl": 2.6679515940486453e-05,
+ "learning_rate": 6.959033804319283e-07,
+ "loss": -0.023484818637371063,
+ "num_tokens": 471647.0,
+ "reward": 0.41836902499198914,
+ "reward_std": 0.3116860091686249,
+ "rewards/true_env_reward_fn/mean": 0.41836902499198914,
+ "rewards/true_env_reward_fn/std": 0.3116860091686249,
+ "step": 116,
+ "step_time": 3.1295652919998247
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 62.375,
+ "completions/mean_terminated_length": 62.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2834057807922363,
+ "epoch": 0.9512195121951219,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.810227154754102e-05,
+ "kl": 1.4841665233689127e-05,
+ "learning_rate": 6.933433136629443e-07,
+ "loss": 7.425555850204546e-07,
+ "num_tokens": 474682.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 117,
+ "step_time": 3.4368692790012574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 63.625,
+ "completions/mean_terminated_length": 63.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.191932499408722,
+ "epoch": 0.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21204856038093567,
+ "kl": 3.64198385796044e-05,
+ "learning_rate": 6.907569807372574e-07,
+ "loss": -0.001312553882598877,
+ "num_tokens": 477027.0,
+ "reward": 0.5300568342208862,
+ "reward_std": 0.2945883274078369,
+ "rewards/true_env_reward_fn/mean": 0.5300568342208862,
+ "rewards/true_env_reward_fn/std": 0.2945883274078369,
+ "step": 118,
+ "step_time": 3.8569856240010267
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 190.0,
+ "completions/max_terminated_length": 190.0,
+ "completions/mean_length": 96.5,
+ "completions/mean_terminated_length": 96.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2401175498962402,
+ "epoch": 0.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011377666669432074,
+ "kl": 1.3742283954343293e-05,
+ "learning_rate": 6.881446132368494e-07,
+ "loss": 6.866695230201003e-07,
+ "num_tokens": 481999.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 119,
+ "step_time": 8.09440958399864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.230682611465454,
+ "epoch": 0.975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22175048291683197,
+ "kl": 1.2522132237791084e-05,
+ "learning_rate": 6.855064450748555e-07,
+ "loss": -0.04083740711212158,
+ "num_tokens": 490884.0,
+ "reward": 0.13476666808128357,
+ "reward_std": 0.2987530529499054,
+ "rewards/true_env_reward_fn/mean": 0.13476666808128357,
+ "rewards/true_env_reward_fn/std": 0.2987530827522278,
+ "step": 120,
+ "step_time": 4.678523641001448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2799639105796814,
+ "epoch": 0.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19926966726779938,
+ "kl": 1.7022688552970067e-05,
+ "learning_rate": 6.828427124746189e-07,
+ "loss": -0.010804429650306702,
+ "num_tokens": 496404.0,
+ "reward": 0.24633333086967468,
+ "reward_std": 0.2454334795475006,
+ "rewards/true_env_reward_fn/mean": 0.24633333086967468,
+ "rewards/true_env_reward_fn/std": 0.2454334795475006,
+ "step": 121,
+ "step_time": 3.98071062300005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 46.875,
+ "completions/mean_terminated_length": 46.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3840235471725464,
+ "epoch": 0.991869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24853822588920593,
+ "kl": 3.688259130285587e-05,
+ "learning_rate": 6.801536539485403e-07,
+ "loss": 0.10205884277820587,
+ "num_tokens": 499767.0,
+ "reward": 0.3045905530452728,
+ "reward_std": 0.262839138507843,
+ "rewards/true_env_reward_fn/mean": 0.3045905530452728,
+ "rewards/true_env_reward_fn/std": 0.262839138507843,
+ "step": 122,
+ "step_time": 3.3792565210005705
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2064164280891418,
+ "epoch": 1.0,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1365528553724289,
+ "kl": 2.434901080050622e-05,
+ "learning_rate": 6.774395102767203e-07,
+ "loss": -0.03472680225968361,
+ "num_tokens": 504906.0,
+ "reward": 0.2722649872303009,
+ "reward_std": 0.2922348082065582,
+ "rewards/true_env_reward_fn/mean": 0.2722649872303009,
+ "rewards/true_env_reward_fn/std": 0.2922348082065582,
+ "step": 123,
+ "step_time": 3.0233660449994204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2819936871528625,
+ "epoch": 1.008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00023045104171615094,
+ "kl": 2.2608143808611203e-05,
+ "learning_rate": 6.747005244854004e-07,
+ "loss": 1.1284330412308918e-06,
+ "num_tokens": 508329.0,
+ "reward": 0.3149532079696655,
+ "reward_std": 0.4275679290294647,
+ "rewards/true_env_reward_fn/mean": 0.3149532079696655,
+ "rewards/true_env_reward_fn/std": 0.4275679886341095,
+ "step": 124,
+ "step_time": 4.01701365199915
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 157.0,
+ "completions/max_terminated_length": 157.0,
+ "completions/mean_length": 80.875,
+ "completions/mean_terminated_length": 80.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1542360186576843,
+ "epoch": 1.016260162601626,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.232049003709108e-05,
+ "kl": 1.291002809011843e-05,
+ "learning_rate": 6.719369418252023e-07,
+ "loss": 6.488799613180163e-07,
+ "num_tokens": 515076.0,
+ "reward": 0.4841846525669098,
+ "reward_std": 0.12780573964118958,
+ "rewards/true_env_reward_fn/mean": 0.4841846525669098,
+ "rewards/true_env_reward_fn/std": 0.12780575454235077,
+ "step": 125,
+ "step_time": 7.240956699999515
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2594389915466309,
+ "epoch": 1.024390243902439,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13091468811035156,
+ "kl": 1.751603304001037e-05,
+ "learning_rate": 6.691490097491675e-07,
+ "loss": -0.033413223922252655,
+ "num_tokens": 520279.0,
+ "reward": 0.28095200657844543,
+ "reward_std": 0.21837711334228516,
+ "rewards/true_env_reward_fn/mean": 0.28095200657844543,
+ "rewards/true_env_reward_fn/std": 0.21837712824344635,
+ "step": 126,
+ "step_time": 3.355879656997786
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 136.0,
+ "completions/max_terminated_length": 136.0,
+ "completions/mean_length": 77.875,
+ "completions/mean_terminated_length": 77.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.3044686317443848,
+ "epoch": 1.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12953205406665802,
+ "kl": 1.8700401597016025e-05,
+ "learning_rate": 6.663369778906008e-07,
+ "loss": 0.03562816232442856,
+ "num_tokens": 524582.0,
+ "reward": 0.4330660402774811,
+ "reward_std": 0.4592672288417816,
+ "rewards/true_env_reward_fn/mean": 0.4330660402774811,
+ "rewards/true_env_reward_fn/std": 0.459267258644104,
+ "step": 127,
+ "step_time": 5.965807722999671
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 117.0,
+ "completions/max_terminated_length": 117.0,
+ "completions/mean_length": 75.375,
+ "completions/mean_terminated_length": 75.375,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1742327809333801,
+ "epoch": 1.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14993594586849213,
+ "kl": 1.1459212601039326e-05,
+ "learning_rate": 6.635010980407174e-07,
+ "loss": 0.03646668791770935,
+ "num_tokens": 526213.0,
+ "reward": 0.7185037136077881,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.7185037136077881,
+ "rewards/true_env_reward_fn/std": 0.1414213627576828,
+ "step": 128,
+ "step_time": 4.9305356690001645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 49.125,
+ "completions/mean_terminated_length": 49.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0784690976142883,
+ "epoch": 1.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16007214784622192,
+ "kl": 1.2491957932070363e-05,
+ "learning_rate": 6.606416241260979e-07,
+ "loss": 0.006608985364437103,
+ "num_tokens": 531862.0,
+ "reward": 0.2934249937534332,
+ "reward_std": 0.2395382523536682,
+ "rewards/true_env_reward_fn/mean": 0.2934249937534332,
+ "rewards/true_env_reward_fn/std": 0.23953823745250702,
+ "step": 129,
+ "step_time": 3.173622508000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.308219850063324,
+ "epoch": 1.056910569105691,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.985446427483112e-05,
+ "kl": 1.2420873190421844e-05,
+ "learning_rate": 6.577588121859508e-07,
+ "loss": 6.241918413252279e-07,
+ "num_tokens": 535957.0,
+ "reward": 0.4817493259906769,
+ "reward_std": 0.029202036559581757,
+ "rewards/true_env_reward_fn/mean": 0.4817493259906769,
+ "rewards/true_env_reward_fn/std": 0.029202038422226906,
+ "step": 130,
+ "step_time": 4.251137947001553
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.0767641067504883,
+ "epoch": 1.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15786457061767578,
+ "kl": 1.8847958926926367e-05,
+ "learning_rate": 6.548529203491875e-07,
+ "loss": -0.0026272237300872803,
+ "num_tokens": 539269.0,
+ "reward": 0.536803662776947,
+ "reward_std": 0.30375123023986816,
+ "rewards/true_env_reward_fn/mean": 0.536803662776947,
+ "rewards/true_env_reward_fn/std": 0.30375123023986816,
+ "step": 131,
+ "step_time": 3.7980547870010923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3295028805732727,
+ "epoch": 1.0731707317073171,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1881481558084488,
+ "kl": 2.0969039724150207e-05,
+ "learning_rate": 6.519242088113085e-07,
+ "loss": 0.08431969583034515,
+ "num_tokens": 545691.0,
+ "reward": 0.24590599536895752,
+ "reward_std": 0.2047487199306488,
+ "rewards/true_env_reward_fn/mean": 0.24590599536895752,
+ "rewards/true_env_reward_fn/std": 0.2047487199306488,
+ "step": 132,
+ "step_time": 4.361092664001262
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2131375670433044,
+ "epoch": 1.08130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13930389285087585,
+ "kl": 1.1046585314034019e-05,
+ "learning_rate": 6.489729398111058e-07,
+ "loss": -0.03801802545785904,
+ "num_tokens": 550295.0,
+ "reward": 0.3215479254722595,
+ "reward_std": 0.1736886352300644,
+ "rewards/true_env_reward_fn/mean": 0.3215479254722595,
+ "rewards/true_env_reward_fn/std": 0.17368865013122559,
+ "step": 133,
+ "step_time": 3.372364626999115
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2786019444465637,
+ "epoch": 1.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12932609021663666,
+ "kl": 1.340499647994875e-05,
+ "learning_rate": 6.459993776071815e-07,
+ "loss": 0.029022663831710815,
+ "num_tokens": 553826.0,
+ "reward": 0.4830188751220703,
+ "reward_std": 0.29014864563941956,
+ "rewards/true_env_reward_fn/mean": 0.4830188751220703,
+ "rewards/true_env_reward_fn/std": 0.29014864563941956,
+ "step": 134,
+ "step_time": 3.215292060998763
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 49.375,
+ "completions/mean_terminated_length": 49.375,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 0.9003906548023224,
+ "epoch": 1.0975609756097562,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15958240628242493,
+ "kl": 2.838099044311093e-05,
+ "learning_rate": 6.430037884542861e-07,
+ "loss": 0.11459673941135406,
+ "num_tokens": 557217.0,
+ "reward": 0.494448184967041,
+ "reward_std": 0.3076546788215637,
+ "rewards/true_env_reward_fn/mean": 0.494448184967041,
+ "rewards/true_env_reward_fn/std": 0.3076546788215637,
+ "step": 135,
+ "step_time": 3.500462582000182
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2928712964057922,
+ "epoch": 1.1056910569105691,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23077522218227386,
+ "kl": 2.251418845844455e-05,
+ "learning_rate": 6.399864405794782e-07,
+ "loss": -0.05874824523925781,
+ "num_tokens": 562421.0,
+ "reward": 0.2385583370923996,
+ "reward_std": 0.23380905389785767,
+ "rewards/true_env_reward_fn/mean": 0.2385583370923996,
+ "rewards/true_env_reward_fn/std": 0.23380906879901886,
+ "step": 136,
+ "step_time": 4.208805245998519
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8736326098442078,
+ "epoch": 1.113821138211382,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.871674071997404e-05,
+ "kl": 1.1485328741400735e-05,
+ "learning_rate": 6.369476041581066e-07,
+ "loss": 5.747077125306532e-07,
+ "num_tokens": 566387.0,
+ "reward": 0.4902166724205017,
+ "reward_std": 0.038254011422395706,
+ "rewards/true_env_reward_fn/mean": 0.4902166724205017,
+ "rewards/true_env_reward_fn/std": 0.038254011422395706,
+ "step": 137,
+ "step_time": 3.981489739000608
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1538971662521362,
+ "epoch": 1.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14676779508590698,
+ "kl": 1.1651037766569061e-05,
+ "learning_rate": 6.338875512896188e-07,
+ "loss": 0.1347643882036209,
+ "num_tokens": 569341.0,
+ "reward": 0.43844783306121826,
+ "reward_std": 0.16067014634609222,
+ "rewards/true_env_reward_fn/mean": 0.43844783306121826,
+ "rewards/true_env_reward_fn/std": 0.16067016124725342,
+ "step": 138,
+ "step_time": 3.689221037999232
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.625,
+ "completions/mean_terminated_length": 54.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1961707472801208,
+ "epoch": 1.1300813008130082,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.16650564968585968,
+ "kl": 1.4349476259667426e-05,
+ "learning_rate": 6.308065559731976e-07,
+ "loss": 0.007910434156656265,
+ "num_tokens": 574046.0,
+ "reward": 0.4596000015735626,
+ "reward_std": 0.07715634256601334,
+ "rewards/true_env_reward_fn/mean": 0.4596000015735626,
+ "rewards/true_env_reward_fn/std": 0.07715633511543274,
+ "step": 139,
+ "step_time": 3.6711935700004688
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2013322114944458,
+ "epoch": 1.1382113821138211,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14415834844112396,
+ "kl": 1.4664098671346437e-05,
+ "learning_rate": 6.277048940832264e-07,
+ "loss": -0.016162052750587463,
+ "num_tokens": 576769.0,
+ "reward": 0.6152583360671997,
+ "reward_std": 0.07727260142564774,
+ "rewards/true_env_reward_fn/mean": 0.6152583360671997,
+ "rewards/true_env_reward_fn/std": 0.07727260142564774,
+ "step": 140,
+ "step_time": 3.5191362610003125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 44.75,
+ "completions/mean_terminated_length": 44.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0287770330905914,
+ "epoch": 1.146341463414634,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.892051457427442e-05,
+ "kl": 1.1797974821092794e-05,
+ "learning_rate": 6.245828433445872e-07,
+ "loss": 5.92092192164273e-07,
+ "num_tokens": 578843.0,
+ "reward": 0.6387845277786255,
+ "reward_std": 0.13867565989494324,
+ "rewards/true_env_reward_fn/mean": 0.6387845277786255,
+ "rewards/true_env_reward_fn/std": 0.13867565989494324,
+ "step": 141,
+ "step_time": 2.5015027329991426
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1590029001235962,
+ "epoch": 1.1544715447154472,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.160966694355011,
+ "kl": 1.4735675904375967e-05,
+ "learning_rate": 6.214406833077937e-07,
+ "loss": 0.0170527845621109,
+ "num_tokens": 583201.0,
+ "reward": 0.36017733812332153,
+ "reward_std": 0.3556094467639923,
+ "rewards/true_env_reward_fn/mean": 0.36017733812332153,
+ "rewards/true_env_reward_fn/std": 0.3556094467639923,
+ "step": 142,
+ "step_time": 3.2783409929998015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.0,
+ "completions/mean_terminated_length": 67.0,
+ "completions/min_length": 59.0,
+ "completions/min_terminated_length": 59.0,
+ "entropy": 1.1985241174697876,
+ "epoch": 1.1626016260162602,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12272457778453827,
+ "kl": 1.7849098185251933e-05,
+ "learning_rate": 6.182786953239593e-07,
+ "loss": -0.0016125142574310303,
+ "num_tokens": 587317.0,
+ "reward": 0.34745320677757263,
+ "reward_std": 0.3954337239265442,
+ "rewards/true_env_reward_fn/mean": 0.34745320677757263,
+ "rewards/true_env_reward_fn/std": 0.3954337537288666,
+ "step": 143,
+ "step_time": 3.9932043310000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 48.0,
+ "completions/max_terminated_length": 48.0,
+ "completions/mean_length": 43.75,
+ "completions/mean_terminated_length": 43.75,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1116944551467896,
+ "epoch": 1.170731707317073,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18435250222682953,
+ "kl": 1.014559029499651e-05,
+ "learning_rate": 6.150971625196048e-07,
+ "loss": 0.009793907403945923,
+ "num_tokens": 590191.0,
+ "reward": 0.4938516616821289,
+ "reward_std": 0.03703190013766289,
+ "rewards/true_env_reward_fn/mean": 0.4938516616821289,
+ "rewards/true_env_reward_fn/std": 0.037031903862953186,
+ "step": 144,
+ "step_time": 2.3663663690022076
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 53.0,
+ "completions/mean_terminated_length": 53.0,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.413200855255127,
+ "epoch": 1.1788617886178863,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14336225390434265,
+ "kl": 2.0541991034406237e-05,
+ "learning_rate": 6.118963697713078e-07,
+ "loss": -0.013927727937698364,
+ "num_tokens": 593671.0,
+ "reward": 0.4619143605232239,
+ "reward_std": 0.3773181140422821,
+ "rewards/true_env_reward_fn/mean": 0.4619143605232239,
+ "rewards/true_env_reward_fn/std": 0.3773181140422821,
+ "step": 145,
+ "step_time": 3.9730388410007436
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 128.0,
+ "completions/max_terminated_length": 128.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 6.0,
+ "completions/min_terminated_length": 6.0,
+ "entropy": 1.08676016330719,
+ "epoch": 1.1869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15555191040039062,
+ "kl": 1.6947700260061538e-05,
+ "learning_rate": 6.086766036801937e-07,
+ "loss": -0.139797180891037,
+ "num_tokens": 601612.0,
+ "reward": 0.3831036686897278,
+ "reward_std": 0.09242849797010422,
+ "rewards/true_env_reward_fn/mean": 0.3831036686897278,
+ "rewards/true_env_reward_fn/std": 0.09242849797010422,
+ "step": 146,
+ "step_time": 6.323679949000507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.5,
+ "completions/mean_terminated_length": 67.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.5055813789367676,
+ "epoch": 1.1951219512195121,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2397669553756714,
+ "kl": 2.704876442294335e-05,
+ "learning_rate": 6.054381525462745e-07,
+ "loss": 0.2738838493824005,
+ "num_tokens": 606712.0,
+ "reward": 0.25339600443840027,
+ "reward_std": 0.3023079037666321,
+ "rewards/true_env_reward_fn/mean": 0.25339600443840027,
+ "rewards/true_env_reward_fn/std": 0.3023079037666321,
+ "step": 147,
+ "step_time": 5.185072233998653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 67.0,
+ "completions/max_terminated_length": 67.0,
+ "completions/mean_length": 47.25,
+ "completions/mean_terminated_length": 47.25,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.135968267917633,
+ "epoch": 1.203252032520325,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2614514231681824,
+ "kl": 3.613240005506668e-05,
+ "learning_rate": 6.021813063426323e-07,
+ "loss": 0.10286401212215424,
+ "num_tokens": 610566.0,
+ "reward": 0.31031692028045654,
+ "reward_std": 0.3124054968357086,
+ "rewards/true_env_reward_fn/mean": 0.31031692028045654,
+ "rewards/true_env_reward_fn/std": 0.312405526638031,
+ "step": 148,
+ "step_time": 3.2177847610000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 52.75,
+ "completions/mean_terminated_length": 52.75,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4589928984642029,
+ "epoch": 1.2113821138211383,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2019941508769989,
+ "kl": 2.1841721718374174e-05,
+ "learning_rate": 5.989063566894572e-07,
+ "loss": 0.010915875434875488,
+ "num_tokens": 615716.0,
+ "reward": 0.31711751222610474,
+ "reward_std": 0.13289952278137207,
+ "rewards/true_env_reward_fn/mean": 0.31711751222610474,
+ "rewards/true_env_reward_fn/std": 0.13289952278137207,
+ "step": 149,
+ "step_time": 4.3804878079990885
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.1892729997634888,
+ "epoch": 1.2195121951219512,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.835455471649766e-05,
+ "kl": 1.3420096820482286e-05,
+ "learning_rate": 5.956135968279332e-07,
+ "loss": 6.646802717114042e-07,
+ "num_tokens": 619439.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 150,
+ "step_time": 3.63938895299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1402934789657593,
+ "epoch": 1.2276422764227641,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010795716661959887,
+ "kl": 1.4652535810455447e-05,
+ "learning_rate": 5.923033215939834e-07,
+ "loss": 7.542968205598299e-07,
+ "num_tokens": 621009.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 151,
+ "step_time": 3.926544339999964
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 64.125,
+ "completions/mean_terminated_length": 64.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.190350890159607,
+ "epoch": 1.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19853363931179047,
+ "kl": 2.269768037876929e-05,
+ "learning_rate": 5.889758273918683e-07,
+ "loss": 0.044217392802238464,
+ "num_tokens": 623994.0,
+ "reward": 0.4411996603012085,
+ "reward_std": 0.2517909109592438,
+ "rewards/true_env_reward_fn/mean": 0.4411996603012085,
+ "rewards/true_env_reward_fn/std": 0.2517908811569214,
+ "step": 152,
+ "step_time": 3.7339736520007136
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.2316884994506836,
+ "epoch": 1.2439024390243902,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.35867181699723e-05,
+ "kl": 1.1579370038816705e-05,
+ "learning_rate": 5.856314121676467e-07,
+ "loss": 5.79387460675207e-07,
+ "num_tokens": 628224.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 153,
+ "step_time": 4.375236807001784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 137.0,
+ "completions/max_terminated_length": 137.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.4028943181037903,
+ "epoch": 1.2520325203252032,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25418931245803833,
+ "kl": 3.6890452065563295e-05,
+ "learning_rate": 5.822703753824966e-07,
+ "loss": 0.25599968433380127,
+ "num_tokens": 631183.0,
+ "reward": 0.38683533668518066,
+ "reward_std": 0.43613559007644653,
+ "rewards/true_env_reward_fn/mean": 0.38683533668518066,
+ "rewards/true_env_reward_fn/std": 0.43613559007644653,
+ "step": 154,
+ "step_time": 6.055355972999678
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.367663562297821,
+ "epoch": 1.2601626016260163,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13331811130046844,
+ "kl": 1.2863993106293492e-05,
+ "learning_rate": 5.788930179859024e-07,
+ "loss": -0.0047044456005096436,
+ "num_tokens": 636230.0,
+ "reward": 0.3489508628845215,
+ "reward_std": 0.13627417385578156,
+ "rewards/true_env_reward_fn/mean": 0.3489508628845215,
+ "rewards/true_env_reward_fn/std": 0.13627417385578156,
+ "step": 155,
+ "step_time": 3.9174396130001696
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 135.0,
+ "completions/max_terminated_length": 135.0,
+ "completions/mean_length": 68.5,
+ "completions/mean_terminated_length": 68.5,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2759611010551453,
+ "epoch": 1.2682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12738144397735596,
+ "kl": 1.5844128029129934e-05,
+ "learning_rate": 5.754996423887061e-07,
+ "loss": -0.011055335402488708,
+ "num_tokens": 640262.0,
+ "reward": 0.344404935836792,
+ "reward_std": 0.16542991995811462,
+ "rewards/true_env_reward_fn/mean": 0.344404935836792,
+ "rewards/true_env_reward_fn/std": 0.16542991995811462,
+ "step": 156,
+ "step_time": 5.706334413998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0904476642608643,
+ "epoch": 1.2764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.733699021628127e-05,
+ "kl": 1.2238857834745431e-05,
+ "learning_rate": 5.720905524360308e-07,
+ "loss": 6.076299996493617e-07,
+ "num_tokens": 645091.0,
+ "reward": 0.4731999933719635,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4731999933719635,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 157,
+ "step_time": 3.7696847109982627
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 60.0,
+ "completions/mean_terminated_length": 60.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.3856677412986755,
+ "epoch": 1.2845528455284554,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18670028448104858,
+ "kl": 1.8415606064081658e-05,
+ "learning_rate": 5.686660533800736e-07,
+ "loss": -0.07078710198402405,
+ "num_tokens": 648179.0,
+ "reward": 0.537517786026001,
+ "reward_std": 0.1451217085123062,
+ "rewards/true_env_reward_fn/mean": 0.537517786026001,
+ "rewards/true_env_reward_fn/std": 0.1451217085123062,
+ "step": 158,
+ "step_time": 3.7075291149994882
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 68.125,
+ "completions/mean_terminated_length": 68.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.121916651725769,
+ "epoch": 1.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11415883898735046,
+ "kl": 1.909901220642496e-05,
+ "learning_rate": 5.652264518527725e-07,
+ "loss": -0.04401372745633125,
+ "num_tokens": 652044.0,
+ "reward": 0.5182899832725525,
+ "reward_std": 0.21869486570358276,
+ "rewards/true_env_reward_fn/mean": 0.5182899832725525,
+ "rewards/true_env_reward_fn/std": 0.21869485080242157,
+ "step": 159,
+ "step_time": 3.8929355969994504
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 143.0,
+ "completions/max_terminated_length": 143.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5205118060112,
+ "epoch": 1.3008130081300813,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.2403375506401062,
+ "kl": 3.7574073758150917e-05,
+ "learning_rate": 5.617720558383508e-07,
+ "loss": 0.26385918259620667,
+ "num_tokens": 656362.0,
+ "reward": 0.41201668977737427,
+ "reward_std": 0.2023741453886032,
+ "rewards/true_env_reward_fn/mean": 0.41201668977737427,
+ "rewards/true_env_reward_fn/std": 0.20237413048744202,
+ "step": 160,
+ "step_time": 6.157555950998358
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3556928038597107,
+ "epoch": 1.3089430894308944,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18135924637317657,
+ "kl": 2.6372636057203636e-05,
+ "learning_rate": 5.583031746457407e-07,
+ "loss": -0.10538280755281448,
+ "num_tokens": 659977.0,
+ "reward": 0.4239906072616577,
+ "reward_std": 0.3287450969219208,
+ "rewards/true_env_reward_fn/mean": 0.4239906072616577,
+ "rewards/true_env_reward_fn/std": 0.3287450969219208,
+ "step": 161,
+ "step_time": 4.709477423999488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 52.5,
+ "completions/mean_terminated_length": 52.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 0.9743769466876984,
+ "epoch": 1.3170731707317074,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.178797647356987,
+ "kl": 1.2532927030406427e-05,
+ "learning_rate": 5.548201188808869e-07,
+ "loss": -0.04164513945579529,
+ "num_tokens": 661409.0,
+ "reward": 0.8041956424713135,
+ "reward_std": 0.1363772451877594,
+ "rewards/true_env_reward_fn/mean": 0.8041956424713135,
+ "rewards/true_env_reward_fn/std": 0.1363772302865982,
+ "step": 162,
+ "step_time": 3.1791253910014348
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.3072250485420227,
+ "epoch": 1.3252032520325203,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16768279671669006,
+ "kl": 1.89386219062726e-05,
+ "learning_rate": 5.513232004189339e-07,
+ "loss": -0.01292814314365387,
+ "num_tokens": 666504.0,
+ "reward": 0.27981066703796387,
+ "reward_std": 0.2949208915233612,
+ "rewards/true_env_reward_fn/mean": 0.27981066703796387,
+ "rewards/true_env_reward_fn/std": 0.2949208915233612,
+ "step": 163,
+ "step_time": 3.778431355000066
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 51.0,
+ "completions/mean_terminated_length": 51.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1984856128692627,
+ "epoch": 1.3333333333333333,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13123764097690582,
+ "kl": 1.9091786271019373e-05,
+ "learning_rate": 5.478127323763027e-07,
+ "loss": 0.035523779690265656,
+ "num_tokens": 671004.0,
+ "reward": 0.2771500051021576,
+ "reward_std": 0.30146247148513794,
+ "rewards/true_env_reward_fn/mean": 0.2771500051021576,
+ "rewards/true_env_reward_fn/std": 0.30146247148513794,
+ "step": 164,
+ "step_time": 3.304021460000513
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 84.875,
+ "completions/mean_terminated_length": 84.875,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2726752758026123,
+ "epoch": 1.3414634146341464,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15594874322414398,
+ "kl": 2.5981638827943243e-05,
+ "learning_rate": 5.442890290826518e-07,
+ "loss": -0.01398652046918869,
+ "num_tokens": 677307.0,
+ "reward": 0.35573017597198486,
+ "reward_std": 0.25944042205810547,
+ "rewards/true_env_reward_fn/mean": 0.35573017597198486,
+ "rewards/true_env_reward_fn/std": 0.25944042205810547,
+ "step": 165,
+ "step_time": 4.972808451999299
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 210.0,
+ "completions/max_terminated_length": 210.0,
+ "completions/mean_length": 103.625,
+ "completions/mean_terminated_length": 103.625,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.0949090719223022,
+ "epoch": 1.3495934959349594,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.07191785424947739,
+ "kl": 1.4828182884230046e-05,
+ "learning_rate": 5.407524060527332e-07,
+ "loss": -0.05063021928071976,
+ "num_tokens": 682076.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 166,
+ "step_time": 8.769379133000257
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1857684254646301,
+ "epoch": 1.3577235772357723,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.923542191041633e-05,
+ "kl": 1.0165251751459436e-05,
+ "learning_rate": 5.37203179958141e-07,
+ "loss": 5.114516170579009e-07,
+ "num_tokens": 685500.0,
+ "reward": 0.5610077381134033,
+ "reward_std": 0.316459059715271,
+ "rewards/true_env_reward_fn/mean": 0.5610077381134033,
+ "rewards/true_env_reward_fn/std": 0.3164590299129486,
+ "step": 167,
+ "step_time": 3.1906087530005607
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 74.25,
+ "completions/mean_terminated_length": 74.25,
+ "completions/min_length": 16.0,
+ "completions/min_terminated_length": 16.0,
+ "entropy": 1.527149498462677,
+ "epoch": 1.3658536585365852,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13164250552654266,
+ "kl": 1.8541333702160046e-05,
+ "learning_rate": 5.33641668598956e-07,
+ "loss": -0.2347300797700882,
+ "num_tokens": 688318.0,
+ "reward": 0.7218117713928223,
+ "reward_std": 0.1818692982196808,
+ "rewards/true_env_reward_fn/mean": 0.7218117713928223,
+ "rewards/true_env_reward_fn/std": 0.1818692833185196,
+ "step": 168,
+ "step_time": 8.50137474999974
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 63.0,
+ "completions/min_terminated_length": 63.0,
+ "entropy": 1.3749513030052185,
+ "epoch": 1.3739837398373984,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13246971368789673,
+ "kl": 1.2620409506780561e-05,
+ "learning_rate": 5.300681908752895e-07,
+ "loss": 0.024534843862056732,
+ "num_tokens": 692541.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 169,
+ "step_time": 3.9512340759993094
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.2032299041748047,
+ "epoch": 1.3821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11185794323682785,
+ "kl": 1.5517784049734473e-05,
+ "learning_rate": 5.264830667587295e-07,
+ "loss": -0.05245225131511688,
+ "num_tokens": 698064.0,
+ "reward": 0.4444866180419922,
+ "reward_std": 0.32400256395339966,
+ "rewards/true_env_reward_fn/mean": 0.4444866180419922,
+ "rewards/true_env_reward_fn/std": 0.32400253415107727,
+ "step": 170,
+ "step_time": 4.471538110999973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 95.0,
+ "completions/mean_terminated_length": 95.0,
+ "completions/min_length": 79.0,
+ "completions/min_terminated_length": 79.0,
+ "entropy": 1.4204387068748474,
+ "epoch": 1.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1646534949541092,
+ "kl": 2.4697198568901513e-05,
+ "learning_rate": 5.228866172636899e-07,
+ "loss": 0.02632315456867218,
+ "num_tokens": 704196.0,
+ "reward": 0.304565966129303,
+ "reward_std": 0.32997164130210876,
+ "rewards/true_env_reward_fn/mean": 0.304565966129303,
+ "rewards/true_env_reward_fn/std": 0.3299716114997864,
+ "step": 171,
+ "step_time": 5.0436168590003945
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1254178285598755,
+ "epoch": 1.3983739837398375,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15058821439743042,
+ "kl": 1.8407325114822015e-05,
+ "learning_rate": 5.192791644186662e-07,
+ "loss": 0.025478817522525787,
+ "num_tokens": 706411.0,
+ "reward": 0.7279239892959595,
+ "reward_std": 0.11376125365495682,
+ "rewards/true_env_reward_fn/mean": 0.7279239892959595,
+ "rewards/true_env_reward_fn/std": 0.11376123130321503,
+ "step": 172,
+ "step_time": 3.2261944119982218
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0894773602485657,
+ "epoch": 1.4065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1687694787979126,
+ "kl": 2.6046765015053097e-05,
+ "learning_rate": 5.156610312374013e-07,
+ "loss": -0.056941211223602295,
+ "num_tokens": 711212.0,
+ "reward": 0.4907146692276001,
+ "reward_std": 0.3376546800136566,
+ "rewards/true_env_reward_fn/mean": 0.4907146692276001,
+ "rewards/true_env_reward_fn/std": 0.337654709815979,
+ "step": 173,
+ "step_time": 3.9498180619993946
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 60.125,
+ "completions/mean_terminated_length": 60.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2238691449165344,
+ "epoch": 1.4146341463414633,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19584441184997559,
+ "kl": 4.8285241064149886e-05,
+ "learning_rate": 5.120325416899629e-07,
+ "loss": 0.0766875222325325,
+ "num_tokens": 715409.0,
+ "reward": 0.4593355059623718,
+ "reward_std": 0.3909546136856079,
+ "rewards/true_env_reward_fn/mean": 0.4593355059623718,
+ "rewards/true_env_reward_fn/std": 0.3909546434879303,
+ "step": 174,
+ "step_time": 4.100519798999812
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 60.25,
+ "completions/mean_terminated_length": 60.25,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1763202548027039,
+ "epoch": 1.4227642276422765,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1448555737733841,
+ "kl": 1.2618989785551094e-05,
+ "learning_rate": 5.08394020673734e-07,
+ "loss": -0.012558378279209137,
+ "num_tokens": 722327.0,
+ "reward": 0.15966665744781494,
+ "reward_std": 0.3235519230365753,
+ "rewards/true_env_reward_fn/mean": 0.15966665744781494,
+ "rewards/true_env_reward_fn/std": 0.32355189323425293,
+ "step": 175,
+ "step_time": 3.875348296000084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2852763533592224,
+ "epoch": 1.4308943089430894,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22303813695907593,
+ "kl": 4.580334098136518e-05,
+ "learning_rate": 5.047457939843227e-07,
+ "loss": -0.09214464575052261,
+ "num_tokens": 726828.0,
+ "reward": 0.25830498337745667,
+ "reward_std": 0.37860655784606934,
+ "rewards/true_env_reward_fn/mean": 0.25830498337745667,
+ "rewards/true_env_reward_fn/std": 0.37860655784606934,
+ "step": 176,
+ "step_time": 4.241473076999682
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.3609731197357178,
+ "epoch": 1.4390243902439024,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.001985745271668e-05,
+ "kl": 1.4942165307729738e-05,
+ "learning_rate": 5.010881882863893e-07,
+ "loss": 7.44550789022469e-07,
+ "num_tokens": 729930.0,
+ "reward": 0.6203632950782776,
+ "reward_std": 0.11898252367973328,
+ "rewards/true_env_reward_fn/mean": 0.6203632950782776,
+ "rewards/true_env_reward_fn/std": 0.11898253113031387,
+ "step": 177,
+ "step_time": 3.3902666960002534
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 59.125,
+ "completions/mean_terminated_length": 59.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2678966522216797,
+ "epoch": 1.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15351757407188416,
+ "kl": 1.6737100395403104e-05,
+ "learning_rate": 4.974215310843967e-07,
+ "loss": 0.041131969541311264,
+ "num_tokens": 732155.0,
+ "reward": 0.7803820371627808,
+ "reward_std": 0.08667682856321335,
+ "rewards/true_env_reward_fn/mean": 0.7803820371627808,
+ "rewards/true_env_reward_fn/std": 0.08667682856321335,
+ "step": 178,
+ "step_time": 3.637111981999624
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.146271526813507,
+ "epoch": 1.4552845528455285,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18466196954250336,
+ "kl": 3.719841197380447e-05,
+ "learning_rate": 4.937461506932859e-07,
+ "loss": 0.029051154851913452,
+ "num_tokens": 735418.0,
+ "reward": 0.40377071499824524,
+ "reward_std": 0.28345924615859985,
+ "rewards/true_env_reward_fn/mean": 0.40377071499824524,
+ "rewards/true_env_reward_fn/std": 0.28345924615859985,
+ "step": 179,
+ "step_time": 3.340555791999577
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 170.0,
+ "completions/max_terminated_length": 170.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.3902945518493652,
+ "epoch": 1.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1020251139998436,
+ "kl": 1.8220500351162627e-05,
+ "learning_rate": 4.900623762090777e-07,
+ "loss": -0.002344265580177307,
+ "num_tokens": 740540.0,
+ "reward": 0.3590222895145416,
+ "reward_std": 0.12487777322530746,
+ "rewards/true_env_reward_fn/mean": 0.3590222895145416,
+ "rewards/true_env_reward_fn/std": 0.12487778067588806,
+ "step": 180,
+ "step_time": 7.219923718001155
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 60.875,
+ "completions/mean_terminated_length": 60.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.517557680606842,
+ "epoch": 1.4715447154471546,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24192294478416443,
+ "kl": 4.1268089262302965e-05,
+ "learning_rate": 4.863705374794055e-07,
+ "loss": 0.09132950007915497,
+ "num_tokens": 744723.0,
+ "reward": 0.23991000652313232,
+ "reward_std": 0.2837013602256775,
+ "rewards/true_env_reward_fn/mean": 0.23991000652313232,
+ "rewards/true_env_reward_fn/std": 0.2837013602256775,
+ "step": 181,
+ "step_time": 3.696339096999509
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.249614655971527,
+ "epoch": 1.4796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1566745787858963,
+ "kl": 2.6629099920683075e-05,
+ "learning_rate": 4.826709650739811e-07,
+ "loss": 0.003972277045249939,
+ "num_tokens": 748979.0,
+ "reward": 0.4935140311717987,
+ "reward_std": 0.41978561878204346,
+ "rewards/true_env_reward_fn/mean": 0.4935140311717987,
+ "rewards/true_env_reward_fn/std": 0.41978561878204346,
+ "step": 182,
+ "step_time": 3.316512920000605
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.1666916608810425,
+ "epoch": 1.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001005820304271765,
+ "kl": 1.2246940059412736e-05,
+ "learning_rate": 4.789639902549948e-07,
+ "loss": 6.330609494398232e-07,
+ "num_tokens": 751323.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 183,
+ "step_time": 3.7099916660008603
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3246222138404846,
+ "epoch": 1.4959349593495934,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.18140996992588043,
+ "kl": 3.1042441150930244e-05,
+ "learning_rate": 4.752499449474535e-07,
+ "loss": -0.022353097796440125,
+ "num_tokens": 755494.0,
+ "reward": 0.4561777412891388,
+ "reward_std": 0.2439236342906952,
+ "rewards/true_env_reward_fn/mean": 0.4561777412891388,
+ "rewards/true_env_reward_fn/std": 0.24392364919185638,
+ "step": 184,
+ "step_time": 3.7916486710000754
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 75.75,
+ "completions/mean_terminated_length": 75.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.2625707983970642,
+ "epoch": 1.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11702944338321686,
+ "kl": 1.503958355897339e-05,
+ "learning_rate": 4.715291617094607e-07,
+ "loss": 0.023916304111480713,
+ "num_tokens": 758432.0,
+ "reward": 0.5015827417373657,
+ "reward_std": 0.1783808320760727,
+ "rewards/true_env_reward_fn/mean": 0.5015827417373657,
+ "rewards/true_env_reward_fn/std": 0.1783808171749115,
+ "step": 185,
+ "step_time": 4.226409274000616
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 25.0,
+ "completions/min_terminated_length": 25.0,
+ "entropy": 1.1460023522377014,
+ "epoch": 1.5121951219512195,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11851620674133301,
+ "kl": 1.3728345948038623e-05,
+ "learning_rate": 4.678019737024387e-07,
+ "loss": 0.0831337422132492,
+ "num_tokens": 764336.0,
+ "reward": 0.19094166159629822,
+ "reward_std": 0.30934420228004456,
+ "rewards/true_env_reward_fn/mean": 0.19094166159629822,
+ "rewards/true_env_reward_fn/std": 0.30934423208236694,
+ "step": 186,
+ "step_time": 3.563357556000483
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1291148662567139,
+ "epoch": 1.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.864256960805506e-05,
+ "kl": 1.081683785741916e-05,
+ "learning_rate": 4.6406871466129704e-07,
+ "loss": 5.412177870312007e-07,
+ "num_tokens": 766608.0,
+ "reward": 0.8357743620872498,
+ "reward_std": 0.11130158603191376,
+ "rewards/true_env_reward_fn/mean": 0.8357743620872498,
+ "rewards/true_env_reward_fn/std": 0.11130159348249435,
+ "step": 187,
+ "step_time": 3.534869859002356
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 189.0,
+ "completions/max_terminated_length": 189.0,
+ "completions/mean_length": 92.0,
+ "completions/mean_terminated_length": 92.0,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.3645328283309937,
+ "epoch": 1.5284552845528454,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18168245255947113,
+ "kl": 2.6857565899263136e-05,
+ "learning_rate": 4.6032971886454956e-07,
+ "loss": -0.005156125873327255,
+ "num_tokens": 774496.0,
+ "reward": 0.11249999701976776,
+ "reward_std": 0.20856082439422607,
+ "rewards/true_env_reward_fn/mean": 0.11249999701976776,
+ "rewards/true_env_reward_fn/std": 0.20856082439422607,
+ "step": 188,
+ "step_time": 8.34005261099992
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1495982110500336,
+ "epoch": 1.5365853658536586,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14486820995807648,
+ "kl": 1.3202762147557223e-05,
+ "learning_rate": 4.5658532110438337e-07,
+ "loss": -0.0010610297322273254,
+ "num_tokens": 777186.0,
+ "reward": 0.5879127383232117,
+ "reward_std": 0.05142820253968239,
+ "rewards/true_env_reward_fn/mean": 0.5879127383232117,
+ "rewards/true_env_reward_fn/std": 0.05142820253968239,
+ "step": 189,
+ "step_time": 3.4856022139993
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.241140365600586,
+ "epoch": 1.5447154471544715,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001250067143701017,
+ "kl": 1.5482702110602986e-05,
+ "learning_rate": 4.52835856656681e-07,
+ "loss": 7.80837922320643e-07,
+ "num_tokens": 779965.0,
+ "reward": 0.6861198544502258,
+ "reward_std": 0.08807206153869629,
+ "rewards/true_env_reward_fn/mean": 0.6861198544502258,
+ "rewards/true_env_reward_fn/std": 0.08807206153869629,
+ "step": 190,
+ "step_time": 3.904181735999373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2944807410240173,
+ "epoch": 1.5528455284552845,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2030111849308014,
+ "kl": 3.585523518268019e-05,
+ "learning_rate": 4.490816612509991e-07,
+ "loss": 0.0143373291939497,
+ "num_tokens": 786140.0,
+ "reward": 0.35173332691192627,
+ "reward_std": 0.18115806579589844,
+ "rewards/true_env_reward_fn/mean": 0.35173332691192627,
+ "rewards/true_env_reward_fn/std": 0.18115808069705963,
+ "step": 191,
+ "step_time": 4.005758510000305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 58.0,
+ "completions/mean_terminated_length": 58.0,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2177271246910095,
+ "epoch": 1.5609756097560976,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11188288033008575,
+ "kl": 2.3622495064046234e-05,
+ "learning_rate": 4.45323071040508e-07,
+ "loss": -0.02369789034128189,
+ "num_tokens": 790424.0,
+ "reward": 0.4888629913330078,
+ "reward_std": 0.23310808837413788,
+ "rewards/true_env_reward_fn/mean": 0.4888629913330078,
+ "rewards/true_env_reward_fn/std": 0.23310808837413788,
+ "step": 192,
+ "step_time": 3.2518814809991454
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.192966103553772,
+ "epoch": 1.5691056910569106,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1378823071718216,
+ "kl": 1.7358055174554465e-05,
+ "learning_rate": 4.4156042257189143e-07,
+ "loss": 0.06256310641765594,
+ "num_tokens": 794521.0,
+ "reward": 0.5219699740409851,
+ "reward_std": 0.06214587390422821,
+ "rewards/true_env_reward_fn/mean": 0.5219699740409851,
+ "rewards/true_env_reward_fn/std": 0.062145888805389404,
+ "step": 193,
+ "step_time": 4.222739491999164
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 63.125,
+ "completions/mean_terminated_length": 63.125,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1770159006118774,
+ "epoch": 1.5772357723577235,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19550059735774994,
+ "kl": 2.4871268578863237e-05,
+ "learning_rate": 4.377940527552125e-07,
+ "loss": 0.05841376632452011,
+ "num_tokens": 798194.0,
+ "reward": 0.42302167415618896,
+ "reward_std": 0.2911272346973419,
+ "rewards/true_env_reward_fn/mean": 0.42302167415618896,
+ "rewards/true_env_reward_fn/std": 0.2911272644996643,
+ "step": 194,
+ "step_time": 3.8170270639984665
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.5244255661964417,
+ "epoch": 1.5853658536585367,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1872202306985855,
+ "kl": 2.5990483663917985e-05,
+ "learning_rate": 4.340242988337462e-07,
+ "loss": -0.044112429022789,
+ "num_tokens": 802802.0,
+ "reward": 0.33745431900024414,
+ "reward_std": 0.22955451905727386,
+ "rewards/true_env_reward_fn/mean": 0.33745431900024414,
+ "rewards/true_env_reward_fn/std": 0.22955450415611267,
+ "step": 195,
+ "step_time": 4.27381555500142
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.345891296863556,
+ "epoch": 1.5934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16891054809093475,
+ "kl": 2.7261638024356216e-05,
+ "learning_rate": 4.3025149835378275e-07,
+ "loss": -0.139386385679245,
+ "num_tokens": 807881.0,
+ "reward": 0.3240283131599426,
+ "reward_std": 0.2803676128387451,
+ "rewards/true_env_reward_fn/mean": 0.3240283131599426,
+ "rewards/true_env_reward_fn/std": 0.2803676128387451,
+ "step": 196,
+ "step_time": 4.981287381999209
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.246802031993866,
+ "epoch": 1.6016260162601625,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2096729576587677,
+ "kl": 3.5958016269432846e-05,
+ "learning_rate": 4.2647598913440264e-07,
+ "loss": -0.02941281348466873,
+ "num_tokens": 812500.0,
+ "reward": 0.4126526415348053,
+ "reward_std": 0.36393746733665466,
+ "rewards/true_env_reward_fn/mean": 0.4126526415348053,
+ "rewards/true_env_reward_fn/std": 0.36393749713897705,
+ "step": 197,
+ "step_time": 3.101726017999681
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4369062185287476,
+ "epoch": 1.6097560975609757,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1862105429172516,
+ "kl": 4.7646244638599455e-05,
+ "learning_rate": 4.2269810923722965e-07,
+ "loss": 0.0521145761013031,
+ "num_tokens": 818718.0,
+ "reward": 0.26869943737983704,
+ "reward_std": 0.1483483463525772,
+ "rewards/true_env_reward_fn/mean": 0.26869943737983704,
+ "rewards/true_env_reward_fn/std": 0.1483483612537384,
+ "step": 198,
+ "step_time": 4.287780451000799
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 21.0,
+ "completions/min_terminated_length": 21.0,
+ "entropy": 1.192937195301056,
+ "epoch": 1.6178861788617886,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2013663798570633,
+ "kl": 1.8760739294521045e-05,
+ "learning_rate": 4.189181969361588e-07,
+ "loss": 0.07236722111701965,
+ "num_tokens": 825728.0,
+ "reward": 0.23110000789165497,
+ "reward_std": 0.23212090134620667,
+ "rewards/true_env_reward_fn/mean": 0.23110000789165497,
+ "rewards/true_env_reward_fn/std": 0.23212090134620667,
+ "step": 199,
+ "step_time": 4.708717262998107
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 72.25,
+ "completions/mean_terminated_length": 72.25,
+ "completions/min_length": 62.0,
+ "completions/min_terminated_length": 62.0,
+ "entropy": 1.3792839050292969,
+ "epoch": 1.6260162601626016,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.7985117463395e-05,
+ "kl": 1.4280476534622721e-05,
+ "learning_rate": 4.1513659068706814e-07,
+ "loss": 7.153485626076872e-07,
+ "num_tokens": 830318.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 200,
+ "step_time": 3.9055351140013954
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0505937337875366,
+ "epoch": 1.6341463414634148,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13188514113426208,
+ "kl": 2.842090088961413e-05,
+ "learning_rate": 4.1135362909751326e-07,
+ "loss": -0.0017508119344711304,
+ "num_tokens": 834933.0,
+ "reward": 0.36545002460479736,
+ "reward_std": 0.24526984989643097,
+ "rewards/true_env_reward_fn/mean": 0.36545002460479736,
+ "rewards/true_env_reward_fn/std": 0.24526986479759216,
+ "step": 201,
+ "step_time": 3.895525625997834
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 163.0,
+ "completions/max_terminated_length": 163.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.3750707507133484,
+ "epoch": 1.6422764227642277,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1798955649137497,
+ "kl": 2.5428611479583196e-05,
+ "learning_rate": 4.075696508964076e-07,
+ "loss": 0.1843666434288025,
+ "num_tokens": 838076.0,
+ "reward": 0.44641831517219543,
+ "reward_std": 0.30040720105171204,
+ "rewards/true_env_reward_fn/mean": 0.44641831517219543,
+ "rewards/true_env_reward_fn/std": 0.30040720105171204,
+ "step": 202,
+ "step_time": 6.74463491500137
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 79.125,
+ "completions/mean_terminated_length": 79.125,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3369249105453491,
+ "epoch": 1.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11249764263629913,
+ "kl": 1.1453126717242412e-05,
+ "learning_rate": 4.0378499490369267e-07,
+ "loss": -0.08210685849189758,
+ "num_tokens": 841933.0,
+ "reward": 0.3737962245941162,
+ "reward_std": 0.13184049725532532,
+ "rewards/true_env_reward_fn/mean": 0.3737962245941162,
+ "rewards/true_env_reward_fn/std": 0.13184049725532532,
+ "step": 203,
+ "step_time": 5.5639925510004105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.5737199783325195,
+ "epoch": 1.6585365853658538,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19394491612911224,
+ "kl": 5.05317857459886e-05,
+ "learning_rate": 4e-07,
+ "loss": 0.05622926354408264,
+ "num_tokens": 847711.0,
+ "reward": 0.10725000500679016,
+ "reward_std": 0.19980257749557495,
+ "rewards/true_env_reward_fn/mean": 0.10725000500679016,
+ "rewards/true_env_reward_fn/std": 0.19980257749557495,
+ "step": 204,
+ "step_time": 4.918089437000162
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 96.0,
+ "completions/max_terminated_length": 96.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.119917094707489,
+ "epoch": 1.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10553985089063644,
+ "kl": 1.2793303994840244e-05,
+ "learning_rate": 3.9621500509630725e-07,
+ "loss": -0.011355768889188766,
+ "num_tokens": 849519.0,
+ "reward": 0.6593211889266968,
+ "reward_std": 0.11862105131149292,
+ "rewards/true_env_reward_fn/mean": 0.6593211889266968,
+ "rewards/true_env_reward_fn/std": 0.11862105131149292,
+ "step": 205,
+ "step_time": 4.127652793999005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 108.0,
+ "completions/max_terminated_length": 108.0,
+ "completions/mean_length": 75.125,
+ "completions/mean_terminated_length": 75.125,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3856809735298157,
+ "epoch": 1.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1911258101463318,
+ "kl": 4.2569914512569085e-05,
+ "learning_rate": 3.9243034910359247e-07,
+ "loss": 0.10561336576938629,
+ "num_tokens": 854156.0,
+ "reward": 0.2516202926635742,
+ "reward_std": 0.22357939183712006,
+ "rewards/true_env_reward_fn/mean": 0.2516202926635742,
+ "rewards/true_env_reward_fn/std": 0.22357939183712006,
+ "step": 206,
+ "step_time": 4.714620994000143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.4012945890426636,
+ "epoch": 1.6829268292682928,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14226125180721283,
+ "kl": 5.4918069963605376e-05,
+ "learning_rate": 3.886463709024868e-07,
+ "loss": 0.012949362397193909,
+ "num_tokens": 856151.0,
+ "reward": 0.6204804182052612,
+ "reward_std": 0.44673967361450195,
+ "rewards/true_env_reward_fn/mean": 0.6204804182052612,
+ "rewards/true_env_reward_fn/std": 0.44673967361450195,
+ "step": 207,
+ "step_time": 4.094810713999323
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.3708943128585815,
+ "epoch": 1.6910569105691056,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12903675436973572,
+ "kl": 1.4145812656352064e-05,
+ "learning_rate": 3.8486340931293187e-07,
+ "loss": -0.07838249206542969,
+ "num_tokens": 859380.0,
+ "reward": 0.6400156021118164,
+ "reward_std": 0.1022576317191124,
+ "rewards/true_env_reward_fn/mean": 0.6400156021118164,
+ "rewards/true_env_reward_fn/std": 0.1022576317191124,
+ "step": 208,
+ "step_time": 4.297900428997309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.75,
+ "completions/mean_terminated_length": 56.75,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2782961130142212,
+ "epoch": 1.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12437833100557327,
+ "kl": 1.4473939245362999e-05,
+ "learning_rate": 3.8108180306384135e-07,
+ "loss": -0.036324724555015564,
+ "num_tokens": 862714.0,
+ "reward": 0.43419933319091797,
+ "reward_std": 0.15345513820648193,
+ "rewards/true_env_reward_fn/mean": 0.43419933319091797,
+ "rewards/true_env_reward_fn/std": 0.15345513820648193,
+ "step": 209,
+ "step_time": 3.47861851900052
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 123.875,
+ "completions/mean_terminated_length": 68.42857360839844,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.5170292258262634,
+ "epoch": 1.7073170731707317,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1543467938899994,
+ "kl": 2.3121557205740828e-05,
+ "learning_rate": 3.7730189076277037e-07,
+ "loss": -0.3810324966907501,
+ "num_tokens": 869789.0,
+ "reward": 0.3795333504676819,
+ "reward_std": 0.13815106451511383,
+ "rewards/true_env_reward_fn/mean": 0.3795333504676819,
+ "rewards/true_env_reward_fn/std": 0.13815106451511383,
+ "step": 210,
+ "step_time": 20.698896928999602
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2324861884117126,
+ "epoch": 1.7154471544715446,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11762262135744095,
+ "kl": 1.9743249595194357e-05,
+ "learning_rate": 3.735240108655973e-07,
+ "loss": -0.09104303270578384,
+ "num_tokens": 872604.0,
+ "reward": 0.5916227102279663,
+ "reward_std": 0.1752101182937622,
+ "rewards/true_env_reward_fn/mean": 0.5916227102279663,
+ "rewards/true_env_reward_fn/std": 0.1752101480960846,
+ "step": 211,
+ "step_time": 5.218213289999767
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.4448966979980469,
+ "epoch": 1.7235772357723578,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1924143135547638,
+ "kl": 3.186432604707079e-05,
+ "learning_rate": 3.697485016462174e-07,
+ "loss": 0.025449808686971664,
+ "num_tokens": 877921.0,
+ "reward": 0.1028124988079071,
+ "reward_std": 0.2254277467727661,
+ "rewards/true_env_reward_fn/mean": 0.1028124988079071,
+ "rewards/true_env_reward_fn/std": 0.22542773187160492,
+ "step": 212,
+ "step_time": 3.415528882000217
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 59.875,
+ "completions/mean_terminated_length": 59.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.185910701751709,
+ "epoch": 1.7317073170731707,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001214127623825334,
+ "kl": 1.3199866316426778e-05,
+ "learning_rate": 3.659757011662538e-07,
+ "loss": 6.776077725589857e-07,
+ "num_tokens": 880344.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 213,
+ "step_time": 3.5156538789997285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 162.0,
+ "completions/max_terminated_length": 162.0,
+ "completions/mean_length": 90.75,
+ "completions/mean_terminated_length": 90.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2697569727897644,
+ "epoch": 1.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1553589552640915,
+ "kl": 1.621047795197228e-05,
+ "learning_rate": 3.622059472447875e-07,
+ "loss": -0.111361563205719,
+ "num_tokens": 885006.0,
+ "reward": 0.5490846633911133,
+ "reward_std": 0.14710450172424316,
+ "rewards/true_env_reward_fn/mean": 0.5490846633911133,
+ "rewards/true_env_reward_fn/std": 0.14710448682308197,
+ "step": 214,
+ "step_time": 7.051423932000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.1904898285865784,
+ "epoch": 1.7479674796747968,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19009968638420105,
+ "kl": 1.6167180092452327e-05,
+ "learning_rate": 3.5843957742810864e-07,
+ "loss": -0.006048411130905151,
+ "num_tokens": 888255.0,
+ "reward": 0.5893601179122925,
+ "reward_std": 0.16021940112113953,
+ "rewards/true_env_reward_fn/mean": 0.5893601179122925,
+ "rewards/true_env_reward_fn/std": 0.16021938621997833,
+ "step": 215,
+ "step_time": 3.0270869319992926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2750649452209473,
+ "epoch": 1.7560975609756098,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1521027386188507,
+ "kl": 4.317680577514693e-05,
+ "learning_rate": 3.5467692895949205e-07,
+ "loss": -0.04247616231441498,
+ "num_tokens": 892490.0,
+ "reward": 0.1841849982738495,
+ "reward_std": 0.30015870928764343,
+ "rewards/true_env_reward_fn/mean": 0.1841849982738495,
+ "rewards/true_env_reward_fn/std": 0.30015870928764343,
+ "step": 216,
+ "step_time": 3.461749838998003
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 62.25,
+ "completions/mean_terminated_length": 62.25,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2730335593223572,
+ "epoch": 1.7642276422764227,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11831019073724747,
+ "kl": 2.1095927877468057e-05,
+ "learning_rate": 3.509183387490009e-07,
+ "loss": -0.051231447607278824,
+ "num_tokens": 894688.0,
+ "reward": 0.49520131945610046,
+ "reward_std": 0.3804744482040405,
+ "rewards/true_env_reward_fn/mean": 0.49520131945610046,
+ "rewards/true_env_reward_fn/std": 0.38047441840171814,
+ "step": 217,
+ "step_time": 3.7184635590019752
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 47.75,
+ "completions/mean_terminated_length": 47.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3015827536582947,
+ "epoch": 1.7723577235772359,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21724702417850494,
+ "kl": 3.562447545846226e-05,
+ "learning_rate": 3.471641433433191e-07,
+ "loss": -0.028775859624147415,
+ "num_tokens": 899066.0,
+ "reward": 0.3376166820526123,
+ "reward_std": 0.21976198256015778,
+ "rewards/true_env_reward_fn/mean": 0.3376166820526123,
+ "rewards/true_env_reward_fn/std": 0.21976199746131897,
+ "step": 218,
+ "step_time": 3.344433074000335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 65.0,
+ "completions/max_terminated_length": 65.0,
+ "completions/mean_length": 53.125,
+ "completions/mean_terminated_length": 53.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1390373706817627,
+ "epoch": 1.7804878048780488,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001077545020962134,
+ "kl": 1.554161144667887e-05,
+ "learning_rate": 3.434146788956166e-07,
+ "loss": 7.676237032683275e-07,
+ "num_tokens": 902263.0,
+ "reward": 0.5682899951934814,
+ "reward_std": 0.1217179074883461,
+ "rewards/true_env_reward_fn/mean": 0.5682899951934814,
+ "rewards/true_env_reward_fn/std": 0.1217179074883461,
+ "step": 219,
+ "step_time": 3.0424018219982827
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.875,
+ "completions/mean_terminated_length": 61.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1991714239120483,
+ "epoch": 1.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11428863555192947,
+ "kl": 1.1265870853094384e-05,
+ "learning_rate": 3.3967028113545045e-07,
+ "loss": -0.02927359938621521,
+ "num_tokens": 906206.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 220,
+ "step_time": 3.5531271640011255
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2546668648719788,
+ "epoch": 1.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1504630595445633,
+ "kl": 1.5663241811125772e-05,
+ "learning_rate": 3.3593128533870314e-07,
+ "loss": 0.04806854575872421,
+ "num_tokens": 909149.0,
+ "reward": 0.49599751830101013,
+ "reward_std": 0.15429075062274933,
+ "rewards/true_env_reward_fn/mean": 0.49599751830101013,
+ "rewards/true_env_reward_fn/std": 0.15429075062274933,
+ "step": 221,
+ "step_time": 3.1345955030010373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1428714394569397,
+ "epoch": 1.8048780487804879,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.310600969707593e-05,
+ "kl": 1.1706195891747484e-05,
+ "learning_rate": 3.321980262975613e-07,
+ "loss": 5.957842290627013e-07,
+ "num_tokens": 914211.0,
+ "reward": 0.3258306384086609,
+ "reward_std": 0.4338511824607849,
+ "rewards/true_env_reward_fn/mean": 0.3258306384086609,
+ "rewards/true_env_reward_fn/std": 0.4338512122631073,
+ "step": 222,
+ "step_time": 3.8445859539988305
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 55.875,
+ "completions/mean_terminated_length": 55.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.1629019975662231,
+ "epoch": 1.8130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010947151895379648,
+ "kl": 1.3530024261854123e-05,
+ "learning_rate": 3.2847083829053923e-07,
+ "loss": 6.723923888785066e-07,
+ "num_tokens": 916850.0,
+ "reward": 0.5956059694290161,
+ "reward_std": 0.09251586347818375,
+ "rewards/true_env_reward_fn/mean": 0.5956059694290161,
+ "rewards/true_env_reward_fn/std": 0.09251587092876434,
+ "step": 223,
+ "step_time": 3.3021794950000185
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.1237311363220215,
+ "epoch": 1.821138211382114,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14958374202251434,
+ "kl": 4.083753810846247e-05,
+ "learning_rate": 3.2475005505254657e-07,
+ "loss": 0.09789139032363892,
+ "num_tokens": 919901.0,
+ "reward": 0.4179220199584961,
+ "reward_std": 0.2486819326877594,
+ "rewards/true_env_reward_fn/mean": 0.4179220199584961,
+ "rewards/true_env_reward_fn/std": 0.2486819475889206,
+ "step": 224,
+ "step_time": 4.954825423001239
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 52.375,
+ "completions/mean_terminated_length": 52.375,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1394256949424744,
+ "epoch": 1.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1480269879102707,
+ "kl": 3.2915593692450784e-05,
+ "learning_rate": 3.210360097450052e-07,
+ "loss": 0.11319300532341003,
+ "num_tokens": 923840.0,
+ "reward": 0.4335233271121979,
+ "reward_std": 0.31114333868026733,
+ "rewards/true_env_reward_fn/mean": 0.4335233271121979,
+ "rewards/true_env_reward_fn/std": 0.31114333868026733,
+ "step": 225,
+ "step_time": 3.777259659000265
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.354669451713562,
+ "epoch": 1.8373983739837398,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13672730326652527,
+ "kl": 2.2323275516100693e-05,
+ "learning_rate": 3.173290349260188e-07,
+ "loss": -0.0521523654460907,
+ "num_tokens": 927882.0,
+ "reward": 0.57341468334198,
+ "reward_std": 0.11576741933822632,
+ "rewards/true_env_reward_fn/mean": 0.57341468334198,
+ "rewards/true_env_reward_fn/std": 0.11576744168996811,
+ "step": 226,
+ "step_time": 3.613498073998926
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1750767230987549,
+ "epoch": 1.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.230674147605896,
+ "kl": 1.800864629331045e-05,
+ "learning_rate": 3.136294625205945e-07,
+ "loss": 0.058730173856019974,
+ "num_tokens": 930225.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 227,
+ "step_time": 3.3819083769976714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 40.875,
+ "completions/mean_terminated_length": 40.875,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.2825847864151,
+ "epoch": 1.8536585365853657,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.30391862988471985,
+ "kl": 2.0106223928451072e-05,
+ "learning_rate": 3.0993762379092235e-07,
+ "loss": -0.10232458263635635,
+ "num_tokens": 933048.0,
+ "reward": 0.45667415857315063,
+ "reward_std": 0.26290765404701233,
+ "rewards/true_env_reward_fn/mean": 0.45667415857315063,
+ "rewards/true_env_reward_fn/std": 0.26290765404701233,
+ "step": 228,
+ "step_time": 2.698590726000475
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 55.0,
+ "completions/max_terminated_length": 55.0,
+ "completions/mean_length": 41.125,
+ "completions/mean_terminated_length": 41.125,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3104369640350342,
+ "epoch": 1.8617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.28224605321884155,
+ "kl": 6.227439189387951e-05,
+ "learning_rate": 3.06253849306714e-07,
+ "loss": -0.008679002523422241,
+ "num_tokens": 938217.0,
+ "reward": 0.2567799985408783,
+ "reward_std": 0.2555168867111206,
+ "rewards/true_env_reward_fn/mean": 0.2567799985408783,
+ "rewards/true_env_reward_fn/std": 0.255516916513443,
+ "step": 229,
+ "step_time": 2.7451177700022527
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.0,
+ "completions/mean_terminated_length": 49.0,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0164751410484314,
+ "epoch": 1.8699186991869918,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011273886048002169,
+ "kl": 1.1985231139988173e-05,
+ "learning_rate": 3.0257846891560323e-07,
+ "loss": 5.987301960885816e-07,
+ "num_tokens": 941049.0,
+ "reward": 0.6971603631973267,
+ "reward_std": 0.2594861686229706,
+ "rewards/true_env_reward_fn/mean": 0.6971603631973267,
+ "rewards/true_env_reward_fn/std": 0.2594861686229706,
+ "step": 230,
+ "step_time": 2.917641182999432
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.1991845965385437,
+ "epoch": 1.8780487804878048,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015363915008492768,
+ "kl": 1.4284144981502322e-05,
+ "learning_rate": 2.989118117136107e-07,
+ "loss": 6.798551339670666e-07,
+ "num_tokens": 945403.0,
+ "reward": 0.31745320558547974,
+ "reward_std": 0.42489534616470337,
+ "rewards/true_env_reward_fn/mean": 0.31745320558547974,
+ "rewards/true_env_reward_fn/std": 0.42489534616470337,
+ "step": 231,
+ "step_time": 5.5585464220002905
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2020843029022217,
+ "epoch": 1.886178861788618,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13684362173080444,
+ "kl": 3.651866154541494e-05,
+ "learning_rate": 2.952542060156773e-07,
+ "loss": 0.025869816541671753,
+ "num_tokens": 949126.0,
+ "reward": 0.3984018564224243,
+ "reward_std": 0.4202974736690521,
+ "rewards/true_env_reward_fn/mean": 0.3984018564224243,
+ "rewards/true_env_reward_fn/std": 0.4202974736690521,
+ "step": 232,
+ "step_time": 3.4383463869999105
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 58.625,
+ "completions/mean_terminated_length": 58.625,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2224581837654114,
+ "epoch": 1.8943089430894309,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20249564945697784,
+ "kl": 3.106597978330683e-05,
+ "learning_rate": 2.9160597932626605e-07,
+ "loss": -0.003915777429938316,
+ "num_tokens": 952815.0,
+ "reward": 0.4141089916229248,
+ "reward_std": 0.3064958453178406,
+ "rewards/true_env_reward_fn/mean": 0.4141089916229248,
+ "rewards/true_env_reward_fn/std": 0.3064958453178406,
+ "step": 233,
+ "step_time": 3.6410487339999236
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1212781071662903,
+ "epoch": 1.9024390243902438,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14502985775470734,
+ "kl": 3.0683338081871625e-05,
+ "learning_rate": 2.879674583100372e-07,
+ "loss": -0.0860406681895256,
+ "num_tokens": 958656.0,
+ "reward": 0.30326664447784424,
+ "reward_std": 0.24800051748752594,
+ "rewards/true_env_reward_fn/mean": 0.30326664447784424,
+ "rewards/true_env_reward_fn/std": 0.24800053238868713,
+ "step": 234,
+ "step_time": 3.8798253620007017
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 60.75,
+ "completions/mean_terminated_length": 60.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.425286054611206,
+ "epoch": 1.910569105691057,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2035023272037506,
+ "kl": 5.512987627298571e-05,
+ "learning_rate": 2.843389687625986e-07,
+ "loss": -0.010439477860927582,
+ "num_tokens": 962638.0,
+ "reward": 0.3157375156879425,
+ "reward_std": 0.46047845482826233,
+ "rewards/true_env_reward_fn/mean": 0.3157375156879425,
+ "rewards/true_env_reward_fn/std": 0.46047845482826233,
+ "step": 235,
+ "step_time": 4.114513064998391
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 120.0,
+ "completions/max_terminated_length": 120.0,
+ "completions/mean_length": 77.625,
+ "completions/mean_terminated_length": 77.625,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4385854601860046,
+ "epoch": 1.91869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12982772290706635,
+ "kl": 1.2591926861205138e-05,
+ "learning_rate": 2.807208355813339e-07,
+ "loss": 0.09093751758337021,
+ "num_tokens": 965755.0,
+ "reward": 0.6140732765197754,
+ "reward_std": 0.27462607622146606,
+ "rewards/true_env_reward_fn/mean": 0.6140732765197754,
+ "rewards/true_env_reward_fn/std": 0.27462607622146606,
+ "step": 236,
+ "step_time": 5.147667763001664
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 71.375,
+ "completions/mean_terminated_length": 71.375,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.3476852178573608,
+ "epoch": 1.9268292682926829,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14480431377887726,
+ "kl": 3.8014684832887724e-05,
+ "learning_rate": 2.771133827363101e-07,
+ "loss": -0.07322391867637634,
+ "num_tokens": 970918.0,
+ "reward": 0.3427826166152954,
+ "reward_std": 0.42430612444877625,
+ "rewards/true_env_reward_fn/mean": 0.3427826166152954,
+ "rewards/true_env_reward_fn/std": 0.42430609464645386,
+ "step": 237,
+ "step_time": 4.67846887900123
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 221.0,
+ "completions/max_terminated_length": 221.0,
+ "completions/mean_length": 94.25,
+ "completions/mean_terminated_length": 94.25,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.3279914855957031,
+ "epoch": 1.934959349593496,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10826019197702408,
+ "kl": 1.3074863090878353e-05,
+ "learning_rate": 2.7351693324127037e-07,
+ "loss": -0.05556309223175049,
+ "num_tokens": 975120.0,
+ "reward": 0.64573073387146,
+ "reward_std": 0.22739914059638977,
+ "rewards/true_env_reward_fn/mean": 0.64573073387146,
+ "rewards/true_env_reward_fn/std": 0.22739915549755096,
+ "step": 238,
+ "step_time": 9.220254810001279
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 66.75,
+ "completions/mean_terminated_length": 66.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3695034384727478,
+ "epoch": 1.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12166028469800949,
+ "kl": 2.6563114261080045e-05,
+ "learning_rate": 2.6993180912471055e-07,
+ "loss": -0.05334407091140747,
+ "num_tokens": 980254.0,
+ "reward": 0.37203267216682434,
+ "reward_std": 0.20089927315711975,
+ "rewards/true_env_reward_fn/mean": 0.37203267216682434,
+ "rewards/true_env_reward_fn/std": 0.20089928805828094,
+ "step": 239,
+ "step_time": 4.224964968001586
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.75,
+ "completions/mean_terminated_length": 53.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.247464120388031,
+ "epoch": 1.951219512195122,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2346416860818863,
+ "kl": 5.1520751185307745e-05,
+ "learning_rate": 2.6635833140104405e-07,
+ "loss": -0.0905834436416626,
+ "num_tokens": 983244.0,
+ "reward": 0.575507640838623,
+ "reward_std": 0.17391785979270935,
+ "rewards/true_env_reward_fn/mean": 0.575507640838623,
+ "rewards/true_env_reward_fn/std": 0.17391787469387054,
+ "step": 240,
+ "step_time": 3.357481237999309
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2175387144088745,
+ "epoch": 1.959349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1870066374540329,
+ "kl": 2.2185965462995227e-05,
+ "learning_rate": 2.6279682004185894e-07,
+ "loss": -0.07365687191486359,
+ "num_tokens": 985574.0,
+ "reward": 0.6635305881500244,
+ "reward_std": 0.1901332288980484,
+ "rewards/true_env_reward_fn/mean": 0.6635305881500244,
+ "rewards/true_env_reward_fn/std": 0.1901332437992096,
+ "step": 241,
+ "step_time": 4.527591582998866
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.25,
+ "completions/mean_terminated_length": 76.25,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2727615237236023,
+ "epoch": 1.967479674796748,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.226071804529056e-05,
+ "kl": 1.4014385214977665e-05,
+ "learning_rate": 2.592475939472668e-07,
+ "loss": 7.015369192231447e-07,
+ "num_tokens": 988868.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 242,
+ "step_time": 4.238274277000528
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 57.5,
+ "completions/mean_terminated_length": 57.5,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1519948840141296,
+ "epoch": 1.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12447996437549591,
+ "kl": 4.700180943473242e-05,
+ "learning_rate": 2.557109709173482e-07,
+ "loss": 0.08819369971752167,
+ "num_tokens": 994068.0,
+ "reward": 0.3696666657924652,
+ "reward_std": 0.18488828837871552,
+ "rewards/true_env_reward_fn/mean": 0.3696666657924652,
+ "rewards/true_env_reward_fn/std": 0.1848883032798767,
+ "step": 243,
+ "step_time": 3.3367313500002638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 115.0,
+ "completions/max_terminated_length": 115.0,
+ "completions/mean_length": 68.25,
+ "completions/mean_terminated_length": 68.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0537148416042328,
+ "epoch": 1.9837398373983741,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17681685090065002,
+ "kl": 3.6240851841284893e-05,
+ "learning_rate": 2.521872676236972e-07,
+ "loss": 0.05281040072441101,
+ "num_tokens": 1000650.0,
+ "reward": 0.12209999561309814,
+ "reward_std": 0.2502918243408203,
+ "rewards/true_env_reward_fn/mean": 0.12209999561309814,
+ "rewards/true_env_reward_fn/std": 0.2502918243408203,
+ "step": 244,
+ "step_time": 5.2957401019993995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 140.0,
+ "completions/max_terminated_length": 140.0,
+ "completions/mean_length": 95.375,
+ "completions/mean_terminated_length": 95.375,
+ "completions/min_length": 61.0,
+ "completions/min_terminated_length": 61.0,
+ "entropy": 1.3126497864723206,
+ "epoch": 1.9918699186991868,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16036106646060944,
+ "kl": 6.061139720259234e-05,
+ "learning_rate": 2.48676799581066e-07,
+ "loss": 0.09418506920337677,
+ "num_tokens": 1005737.0,
+ "reward": 0.31175702810287476,
+ "reward_std": 0.38867074251174927,
+ "rewards/true_env_reward_fn/mean": 0.31175702810287476,
+ "rewards/true_env_reward_fn/std": 0.38867077231407166,
+ "step": 245,
+ "step_time": 6.259088058999623
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 48.5,
+ "completions/mean_terminated_length": 48.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.21333646774292,
+ "epoch": 2.0,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23997871577739716,
+ "kl": 3.1378609492094256e-05,
+ "learning_rate": 2.4517988111911313e-07,
+ "loss": 0.010592922568321228,
+ "num_tokens": 1010869.0,
+ "reward": 0.33381664752960205,
+ "reward_std": 0.18213039636611938,
+ "rewards/true_env_reward_fn/mean": 0.33381664752960205,
+ "rewards/true_env_reward_fn/std": 0.18213039636611938,
+ "step": 246,
+ "step_time": 3.111915630997828
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 58.0,
+ "completions/min_terminated_length": 58.0,
+ "entropy": 1.3962982892990112,
+ "epoch": 2.008130081300813,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21355785429477692,
+ "kl": 3.789625407080166e-05,
+ "learning_rate": 2.4169682535425927e-07,
+ "loss": 0.025682777166366577,
+ "num_tokens": 1014876.0,
+ "reward": 0.35749268531799316,
+ "reward_std": 0.29738906025886536,
+ "rewards/true_env_reward_fn/mean": 0.35749268531799316,
+ "rewards/true_env_reward_fn/std": 0.29738909006118774,
+ "step": 247,
+ "step_time": 3.35338095500083
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 73.875,
+ "completions/mean_terminated_length": 73.875,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.463137686252594,
+ "epoch": 2.016260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18188992142677307,
+ "kl": 1.746804719005013e-05,
+ "learning_rate": 2.382279441616492e-07,
+ "loss": -0.17857304215431213,
+ "num_tokens": 1018383.0,
+ "reward": 0.5329012274742126,
+ "reward_std": 0.055823445320129395,
+ "rewards/true_env_reward_fn/mean": 0.5329012274742126,
+ "rewards/true_env_reward_fn/std": 0.05582345277070999,
+ "step": 248,
+ "step_time": 5.210386754000865
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 211.0,
+ "completions/max_terminated_length": 211.0,
+ "completions/mean_length": 79.25,
+ "completions/mean_terminated_length": 79.25,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.4478936195373535,
+ "epoch": 2.024390243902439,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.744662434561178e-05,
+ "kl": 1.3336490155779757e-05,
+ "learning_rate": 2.3477354814722762e-07,
+ "loss": 6.725406365148956e-07,
+ "num_tokens": 1022753.0,
+ "reward": 0.5905972719192505,
+ "reward_std": 0.15080371499061584,
+ "rewards/true_env_reward_fn/mean": 0.5905972719192505,
+ "rewards/true_env_reward_fn/std": 0.15080371499061584,
+ "step": 249,
+ "step_time": 8.798317029002646
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 64.75,
+ "completions/mean_terminated_length": 64.75,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.37166029214859,
+ "epoch": 2.032520325203252,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1542455554008484,
+ "kl": 2.0379054603836266e-05,
+ "learning_rate": 2.313339466199264e-07,
+ "loss": -0.037539318203926086,
+ "num_tokens": 1025971.0,
+ "reward": 0.6065863966941833,
+ "reward_std": 0.032470256090164185,
+ "rewards/true_env_reward_fn/mean": 0.6065863966941833,
+ "rewards/true_env_reward_fn/std": 0.032470256090164185,
+ "step": 250,
+ "step_time": 4.096263454999644
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 60.5,
+ "completions/mean_terminated_length": 60.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.4042693972587585,
+ "epoch": 2.040650406504065,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.163822203874588,
+ "kl": 3.525477495713858e-05,
+ "learning_rate": 2.2790944756396916e-07,
+ "loss": 0.03408379852771759,
+ "num_tokens": 1029415.0,
+ "reward": 0.37829869985580444,
+ "reward_std": 0.2773255407810211,
+ "rewards/true_env_reward_fn/mean": 0.37829869985580444,
+ "rewards/true_env_reward_fn/std": 0.2773255407810211,
+ "step": 251,
+ "step_time": 3.9464334140011488
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 181.0,
+ "completions/max_terminated_length": 181.0,
+ "completions/mean_length": 73.75,
+ "completions/mean_terminated_length": 73.75,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2268111109733582,
+ "epoch": 2.048780487804878,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1961166262626648,
+ "kl": 3.528672823449597e-05,
+ "learning_rate": 2.2450035761129391e-07,
+ "loss": 0.2999379336833954,
+ "num_tokens": 1032213.0,
+ "reward": 0.6772161722183228,
+ "reward_std": 0.31218820810317993,
+ "rewards/true_env_reward_fn/mean": 0.6772161722183228,
+ "rewards/true_env_reward_fn/std": 0.31218820810317993,
+ "step": 252,
+ "step_time": 7.437029113001699
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.360656499862671,
+ "epoch": 2.0569105691056913,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24334783852100372,
+ "kl": 5.1042834456893615e-05,
+ "learning_rate": 2.2110698201409787e-07,
+ "loss": 0.05509951710700989,
+ "num_tokens": 1038598.0,
+ "reward": 0.2947666645050049,
+ "reward_std": 0.19891902804374695,
+ "rewards/true_env_reward_fn/mean": 0.2947666645050049,
+ "rewards/true_env_reward_fn/std": 0.19891902804374695,
+ "step": 253,
+ "step_time": 3.7359043900014512
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 74.75,
+ "completions/mean_terminated_length": 74.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4314632415771484,
+ "epoch": 2.065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1960419863462448,
+ "kl": 6.17889963905327e-05,
+ "learning_rate": 2.1772962461750342e-07,
+ "loss": 0.0954262986779213,
+ "num_tokens": 1043268.0,
+ "reward": 0.4036714732646942,
+ "reward_std": 0.42137831449508667,
+ "rewards/true_env_reward_fn/mean": 0.4036714732646942,
+ "rewards/true_env_reward_fn/std": 0.42137834429740906,
+ "step": 254,
+ "step_time": 5.565175547999388
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0521443486213684,
+ "epoch": 2.073170731707317,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.464648271910846e-05,
+ "kl": 1.2018902907584561e-05,
+ "learning_rate": 2.1436858783235338e-07,
+ "loss": 6.008343689245521e-07,
+ "num_tokens": 1046517.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 255,
+ "step_time": 4.379171047001364
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.091518223285675,
+ "epoch": 2.08130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 9.263061656383798e-05,
+ "kl": 1.4573892713087844e-05,
+ "learning_rate": 2.110241726081317e-07,
+ "loss": 7.304333848878741e-07,
+ "num_tokens": 1050439.0,
+ "reward": 0.6024306416511536,
+ "reward_std": 0.13815335929393768,
+ "rewards/true_env_reward_fn/mean": 0.6024306416511536,
+ "rewards/true_env_reward_fn/std": 0.13815335929393768,
+ "step": 256,
+ "step_time": 3.8379976090000127
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2301559448242188,
+ "epoch": 2.089430894308943,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.17180238664150238,
+ "kl": 2.236898035334889e-05,
+ "learning_rate": 2.076966784060165e-07,
+ "loss": 0.023751959204673767,
+ "num_tokens": 1054578.0,
+ "reward": 0.4112047851085663,
+ "reward_std": 0.05330020561814308,
+ "rewards/true_env_reward_fn/mean": 0.4112047851085663,
+ "rewards/true_env_reward_fn/std": 0.05330020561814308,
+ "step": 257,
+ "step_time": 4.424114469000415
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 52.125,
+ "completions/mean_terminated_length": 52.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2734522223472595,
+ "epoch": 2.097560975609756,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2274128645658493,
+ "kl": 5.3426387239596806e-05,
+ "learning_rate": 2.043864031720667e-07,
+ "loss": 0.05165906995534897,
+ "num_tokens": 1060579.0,
+ "reward": 0.2129499912261963,
+ "reward_std": 0.2476053088903427,
+ "rewards/true_env_reward_fn/mean": 0.2129499912261963,
+ "rewards/true_env_reward_fn/std": 0.2476053088903427,
+ "step": 258,
+ "step_time": 3.4830677139998443
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 54.375,
+ "completions/mean_terminated_length": 54.375,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2013150453567505,
+ "epoch": 2.105691056910569,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20796725153923035,
+ "kl": 8.34841157484334e-05,
+ "learning_rate": 2.0109364331054297e-07,
+ "loss": 0.04251064360141754,
+ "num_tokens": 1065318.0,
+ "reward": 0.2038009911775589,
+ "reward_std": 0.3393669128417969,
+ "rewards/true_env_reward_fn/mean": 0.2038009911775589,
+ "rewards/true_env_reward_fn/std": 0.3393669128417969,
+ "step": 259,
+ "step_time": 3.34712773299907
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.3005307912826538,
+ "epoch": 2.113821138211382,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2342396229505539,
+ "kl": 2.9608699151140172e-05,
+ "learning_rate": 1.9781869365736777e-07,
+ "loss": -0.11542908847332001,
+ "num_tokens": 1068352.0,
+ "reward": 0.5755212306976318,
+ "reward_std": 0.1629202961921692,
+ "rewards/true_env_reward_fn/mean": 0.5755212306976318,
+ "rewards/true_env_reward_fn/std": 0.1629202961921692,
+ "step": 260,
+ "step_time": 3.0931306170004973
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1381222009658813,
+ "epoch": 2.1219512195121952,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13169968128204346,
+ "kl": 1.4705466128361877e-05,
+ "learning_rate": 1.9456184745372558e-07,
+ "loss": -0.026440951973199844,
+ "num_tokens": 1070847.0,
+ "reward": 0.5704532265663147,
+ "reward_std": 0.13928835093975067,
+ "rewards/true_env_reward_fn/mean": 0.5704532265663147,
+ "rewards/true_env_reward_fn/std": 0.13928835093975067,
+ "step": 261,
+ "step_time": 3.8980969309996
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.241390347480774,
+ "epoch": 2.130081300813008,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010450400441186503,
+ "kl": 1.6406540453317575e-05,
+ "learning_rate": 1.9132339631980622e-07,
+ "loss": 8.202600838558283e-07,
+ "num_tokens": 1076153.0,
+ "reward": 0.41493332386016846,
+ "reward_std": 0.06228968873620033,
+ "rewards/true_env_reward_fn/mean": 0.41493332386016846,
+ "rewards/true_env_reward_fn/std": 0.06228969246149063,
+ "step": 262,
+ "step_time": 3.6601423579995753
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 81.0,
+ "completions/mean_terminated_length": 81.0,
+ "completions/min_length": 67.0,
+ "completions/min_terminated_length": 67.0,
+ "entropy": 1.1326860189437866,
+ "epoch": 2.138211382113821,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001754833065206185,
+ "kl": 1.4390577689482598e-05,
+ "learning_rate": 1.881036302286923e-07,
+ "loss": 7.2446778176527e-07,
+ "num_tokens": 1080537.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 263,
+ "step_time": 4.190891189999093
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 103.0,
+ "completions/max_terminated_length": 103.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.353486955165863,
+ "epoch": 2.1463414634146343,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.215454563498497,
+ "kl": 6.859865607111715e-05,
+ "learning_rate": 1.8490283748039515e-07,
+ "loss": 0.0181141197681427,
+ "num_tokens": 1084056.0,
+ "reward": 0.3049938380718231,
+ "reward_std": 0.4605039656162262,
+ "rewards/true_env_reward_fn/mean": 0.3049938380718231,
+ "rewards/true_env_reward_fn/std": 0.4605039954185486,
+ "step": 264,
+ "step_time": 4.504906432999633
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 127.0,
+ "completions/max_terminated_length": 127.0,
+ "completions/mean_length": 70.375,
+ "completions/mean_terminated_length": 70.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0113105773925781,
+ "epoch": 2.154471544715447,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10072485357522964,
+ "kl": 1.4604076568502933e-05,
+ "learning_rate": 1.8172130467604085e-07,
+ "loss": -0.041721273213624954,
+ "num_tokens": 1090171.0,
+ "reward": 0.47745320200920105,
+ "reward_std": 0.2929421067237854,
+ "rewards/true_env_reward_fn/mean": 0.47745320200920105,
+ "rewards/true_env_reward_fn/std": 0.2929421067237854,
+ "step": 265,
+ "step_time": 6.299696521999067
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 57.375,
+ "completions/mean_terminated_length": 57.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3366597294807434,
+ "epoch": 2.16260162601626,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2297964245080948,
+ "kl": 5.677436638507061e-05,
+ "learning_rate": 1.785593166922062e-07,
+ "loss": 0.20361776649951935,
+ "num_tokens": 1094358.0,
+ "reward": 0.3835672438144684,
+ "reward_std": 0.4339357614517212,
+ "rewards/true_env_reward_fn/mean": 0.3835672438144684,
+ "rewards/true_env_reward_fn/std": 0.4339357912540436,
+ "step": 266,
+ "step_time": 4.192992550000781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.4040917754173279,
+ "epoch": 2.1707317073170733,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1913405954837799,
+ "kl": 2.2514723241329193e-05,
+ "learning_rate": 1.7541715665541276e-07,
+ "loss": -0.12278837710618973,
+ "num_tokens": 1099685.0,
+ "reward": 0.5316476821899414,
+ "reward_std": 0.1867343932390213,
+ "rewards/true_env_reward_fn/mean": 0.5316476821899414,
+ "rewards/true_env_reward_fn/std": 0.1867344230413437,
+ "step": 267,
+ "step_time": 4.577502725998784
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2707499265670776,
+ "epoch": 2.178861788617886,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1252160668373108,
+ "kl": 2.1676510186807718e-05,
+ "learning_rate": 1.722951059167737e-07,
+ "loss": 0.030697450041770935,
+ "num_tokens": 1103244.0,
+ "reward": 0.47949954867362976,
+ "reward_std": 0.10323704034090042,
+ "rewards/true_env_reward_fn/mean": 0.47949954867362976,
+ "rewards/true_env_reward_fn/std": 0.10323705524206161,
+ "step": 268,
+ "step_time": 4.3898782989999745
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3778526186943054,
+ "epoch": 2.186991869918699,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19924476742744446,
+ "kl": 5.189802323002368e-05,
+ "learning_rate": 1.6919344402680231e-07,
+ "loss": 0.03015017881989479,
+ "num_tokens": 1108318.0,
+ "reward": 0.12559716403484344,
+ "reward_std": 0.22310735285282135,
+ "rewards/true_env_reward_fn/mean": 0.12559716403484344,
+ "rewards/true_env_reward_fn/std": 0.22310735285282135,
+ "step": 269,
+ "step_time": 4.379851057999986
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.2916911840438843,
+ "epoch": 2.1951219512195124,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18381677567958832,
+ "kl": 2.593698127384414e-05,
+ "learning_rate": 1.6611244871038116e-07,
+ "loss": 0.025029506534337997,
+ "num_tokens": 1116025.0,
+ "reward": 0.05200198292732239,
+ "reward_std": 0.276480108499527,
+ "rewards/true_env_reward_fn/mean": 0.05200198292732239,
+ "rewards/true_env_reward_fn/std": 0.276480108499527,
+ "step": 270,
+ "step_time": 3.6788300769985653
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 215.0,
+ "completions/max_terminated_length": 215.0,
+ "completions/mean_length": 94.625,
+ "completions/mean_terminated_length": 94.625,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.2625537514686584,
+ "epoch": 2.203252032520325,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.09628148376941681,
+ "kl": 2.636932003952097e-05,
+ "learning_rate": 1.6305239584189344e-07,
+ "loss": 0.009904414415359497,
+ "num_tokens": 1120434.0,
+ "reward": 0.5924437046051025,
+ "reward_std": 0.39917245507240295,
+ "rewards/true_env_reward_fn/mean": 0.5924437046051025,
+ "rewards/true_env_reward_fn/std": 0.39917245507240295,
+ "step": 271,
+ "step_time": 9.09279120499923
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.2491654753684998,
+ "epoch": 2.2113821138211383,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015083610196597874,
+ "kl": 1.4522283436235739e-05,
+ "learning_rate": 1.6001355942052182e-07,
+ "loss": 7.247089683914965e-07,
+ "num_tokens": 1122349.0,
+ "reward": 0.812765896320343,
+ "reward_std": 0.04731824994087219,
+ "rewards/true_env_reward_fn/mean": 0.812765896320343,
+ "rewards/true_env_reward_fn/std": 0.04731824994087219,
+ "step": 272,
+ "step_time": 3.7730076539992297
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 71.25,
+ "completions/mean_terminated_length": 71.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1462301015853882,
+ "epoch": 2.2195121951219514,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00012917000276502222,
+ "kl": 1.3649782886204775e-05,
+ "learning_rate": 1.569962115457138e-07,
+ "loss": 6.695274805679219e-07,
+ "num_tokens": 1124831.0,
+ "reward": 0.7329437732696533,
+ "reward_std": 0.22123214602470398,
+ "rewards/true_env_reward_fn/mean": 0.7329437732696533,
+ "rewards/true_env_reward_fn/std": 0.22123214602470398,
+ "step": 273,
+ "step_time": 3.907510233000721
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 175.0,
+ "completions/max_terminated_length": 175.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2922418713569641,
+ "epoch": 2.227642276422764,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15563185513019562,
+ "kl": 3.604595076467376e-05,
+ "learning_rate": 1.5400062239281858e-07,
+ "loss": -0.034219659864902496,
+ "num_tokens": 1128837.0,
+ "reward": 0.46255773305892944,
+ "reward_std": 0.24893923103809357,
+ "rewards/true_env_reward_fn/mean": 0.46255773305892944,
+ "rewards/true_env_reward_fn/std": 0.24893923103809357,
+ "step": 274,
+ "step_time": 7.502110859999448
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.2709790468215942,
+ "epoch": 2.2357723577235773,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24958185851573944,
+ "kl": 6.180045966175385e-05,
+ "learning_rate": 1.5102706018889428e-07,
+ "loss": -0.10148808360099792,
+ "num_tokens": 1132891.0,
+ "reward": 0.4240284562110901,
+ "reward_std": 0.43752968311309814,
+ "rewards/true_env_reward_fn/mean": 0.4240284562110901,
+ "rewards/true_env_reward_fn/std": 0.43752965331077576,
+ "step": 275,
+ "step_time": 3.251475233999372
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 55.5,
+ "completions/mean_terminated_length": 55.5,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.2637454867362976,
+ "epoch": 2.2439024390243905,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.439944602083415e-05,
+ "kl": 1.127877567341784e-05,
+ "learning_rate": 1.4807579118869146e-07,
+ "loss": 5.64579522688291e-07,
+ "num_tokens": 1137611.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 276,
+ "step_time": 3.311975311999049
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.1065265536308289,
+ "epoch": 2.252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13068579137325287,
+ "kl": 4.4293181417742744e-05,
+ "learning_rate": 1.4514707965081262e-07,
+ "loss": 0.04848391190171242,
+ "num_tokens": 1142087.0,
+ "reward": 0.2852628827095032,
+ "reward_std": 0.23009054362773895,
+ "rewards/true_env_reward_fn/mean": 0.2852628827095032,
+ "rewards/true_env_reward_fn/std": 0.23009057343006134,
+ "step": 277,
+ "step_time": 3.9521008399988204
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 168.0,
+ "completions/max_terminated_length": 168.0,
+ "completions/mean_length": 104.5,
+ "completions/mean_terminated_length": 104.5,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.5374161005020142,
+ "epoch": 2.2601626016260163,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.129844531416893,
+ "kl": 3.9368313082377426e-05,
+ "learning_rate": 1.4224118781404917e-07,
+ "loss": -0.1482687145471573,
+ "num_tokens": 1147731.0,
+ "reward": 0.43388551473617554,
+ "reward_std": 0.39525240659713745,
+ "rewards/true_env_reward_fn/mean": 0.43388551473617554,
+ "rewards/true_env_reward_fn/std": 0.39525243639945984,
+ "step": 278,
+ "step_time": 7.55689369099855
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.0975646376609802,
+ "epoch": 2.2682926829268295,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1335010528564453,
+ "kl": 2.2682882445224095e-05,
+ "learning_rate": 1.3935837587390214e-07,
+ "loss": -0.009766265749931335,
+ "num_tokens": 1151246.0,
+ "reward": 0.37655720114707947,
+ "reward_std": 0.37258440256118774,
+ "rewards/true_env_reward_fn/mean": 0.37655720114707947,
+ "rewards/true_env_reward_fn/std": 0.37258440256118774,
+ "step": 279,
+ "step_time": 3.8792882219986495
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 63.5,
+ "completions/mean_terminated_length": 63.5,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.1846895217895508,
+ "epoch": 2.2764227642276422,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.138702854514122,
+ "kl": 2.0143882466072682e-05,
+ "learning_rate": 1.3649890195928254e-07,
+ "loss": 0.013748884201049805,
+ "num_tokens": 1153222.0,
+ "reward": 0.8218116760253906,
+ "reward_std": 0.1293545663356781,
+ "rewards/true_env_reward_fn/mean": 0.8218116760253906,
+ "rewards/true_env_reward_fn/std": 0.1293545812368393,
+ "step": 280,
+ "step_time": 3.42550413599929
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 64.0,
+ "completions/mean_terminated_length": 64.0,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3451175689697266,
+ "epoch": 2.2845528455284554,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1508527249097824,
+ "kl": 1.6968931049632374e-05,
+ "learning_rate": 1.336630221093991e-07,
+ "loss": 0.04001428931951523,
+ "num_tokens": 1157606.0,
+ "reward": 0.4994586706161499,
+ "reward_std": 0.10593737661838531,
+ "rewards/true_env_reward_fn/mean": 0.4994586706161499,
+ "rewards/true_env_reward_fn/std": 0.10593737661838531,
+ "step": 281,
+ "step_time": 3.914840199000537
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2436452507972717,
+ "epoch": 2.292682926829268,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12380471080541611,
+ "kl": 4.350653580331709e-05,
+ "learning_rate": 1.3085099025083245e-07,
+ "loss": -0.029160797595977783,
+ "num_tokens": 1162012.0,
+ "reward": 0.5066306591033936,
+ "reward_std": 0.28914663195610046,
+ "rewards/true_env_reward_fn/mean": 0.5066306591033936,
+ "rewards/true_env_reward_fn/std": 0.28914666175842285,
+ "step": 282,
+ "step_time": 3.9093819319987233
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 59.375,
+ "completions/mean_terminated_length": 59.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.10621577501297,
+ "epoch": 2.3008130081300813,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001239393459400162,
+ "kl": 1.6069413049990544e-05,
+ "learning_rate": 1.2806305817479771e-07,
+ "loss": 8.174432082341809e-07,
+ "num_tokens": 1165367.0,
+ "reward": 0.6557307243347168,
+ "reward_std": 0.2151959389448166,
+ "rewards/true_env_reward_fn/mean": 0.6557307243347168,
+ "rewards/true_env_reward_fn/std": 0.21519595384597778,
+ "step": 283,
+ "step_time": 3.8117841049988783
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 66.0,
+ "completions/mean_terminated_length": 66.0,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.382466197013855,
+ "epoch": 2.3089430894308944,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1446879506111145,
+ "kl": 3.576014205464162e-05,
+ "learning_rate": 1.2529947551459964e-07,
+ "loss": -0.04731176793575287,
+ "num_tokens": 1169243.0,
+ "reward": 0.496622234582901,
+ "reward_std": 0.17860308289527893,
+ "rewards/true_env_reward_fn/mean": 0.496622234582901,
+ "rewards/true_env_reward_fn/std": 0.17860306799411774,
+ "step": 284,
+ "step_time": 3.649135475998264
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 151.0,
+ "completions/max_terminated_length": 151.0,
+ "completions/mean_length": 61.375,
+ "completions/mean_terminated_length": 61.375,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1946157813072205,
+ "epoch": 2.317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10612626373767853,
+ "kl": 2.0257402866263874e-05,
+ "learning_rate": 1.2256048972327967e-07,
+ "loss": -0.029511645436286926,
+ "num_tokens": 1173594.0,
+ "reward": 0.5235810279846191,
+ "reward_std": 0.2810492217540741,
+ "rewards/true_env_reward_fn/mean": 0.5235810279846191,
+ "rewards/true_env_reward_fn/std": 0.2810492217540741,
+ "step": 285,
+ "step_time": 6.7011265000001
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 95.0,
+ "completions/max_terminated_length": 95.0,
+ "completions/mean_length": 68.75,
+ "completions/mean_terminated_length": 68.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2879216074943542,
+ "epoch": 2.3252032520325203,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12234477698802948,
+ "kl": 1.4488860415440286e-05,
+ "learning_rate": 1.1984634605145978e-07,
+ "loss": -0.05451745539903641,
+ "num_tokens": 1176384.0,
+ "reward": 0.6496104001998901,
+ "reward_std": 0.3368098735809326,
+ "rewards/true_env_reward_fn/mean": 0.6496104001998901,
+ "rewards/true_env_reward_fn/std": 0.3368098735809326,
+ "step": 286,
+ "step_time": 4.18911992899848
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 70.875,
+ "completions/mean_terminated_length": 70.875,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0649144053459167,
+ "epoch": 2.3333333333333335,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1467445194721222,
+ "kl": 3.947542245441582e-05,
+ "learning_rate": 1.17157287525381e-07,
+ "loss": -0.029090911149978638,
+ "num_tokens": 1179651.0,
+ "reward": 0.4754716753959656,
+ "reward_std": 0.2500669062137604,
+ "rewards/true_env_reward_fn/mean": 0.4754716753959656,
+ "rewards/true_env_reward_fn/std": 0.2500669062137604,
+ "step": 287,
+ "step_time": 4.5810332049986755
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 1.002779871225357,
+ "epoch": 2.341463414634146,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12533478438854218,
+ "kl": 2.3054610210238025e-05,
+ "learning_rate": 1.1449355492514437e-07,
+ "loss": -0.04500773549079895,
+ "num_tokens": 1181547.0,
+ "reward": 0.7334807515144348,
+ "reward_std": 0.12557923793792725,
+ "rewards/true_env_reward_fn/mean": 0.7334807515144348,
+ "rewards/true_env_reward_fn/std": 0.12557923793792725,
+ "step": 288,
+ "step_time": 3.8661079009998502
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 294.0,
+ "completions/max_terminated_length": 294.0,
+ "completions/mean_length": 81.25,
+ "completions/mean_terminated_length": 81.25,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2280530333518982,
+ "epoch": 2.3495934959349594,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23682235181331635,
+ "kl": 5.290110129863024e-05,
+ "learning_rate": 1.1185538676315052e-07,
+ "loss": 0.5538168549537659,
+ "num_tokens": 1183973.0,
+ "reward": 0.5785378217697144,
+ "reward_std": 0.29215970635414124,
+ "rewards/true_env_reward_fn/mean": 0.5785378217697144,
+ "rewards/true_env_reward_fn/std": 0.2921597361564636,
+ "step": 289,
+ "step_time": 11.744046860998424
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.264986515045166,
+ "epoch": 2.3577235772357725,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001335782726528123,
+ "kl": 1.7416054106433876e-05,
+ "learning_rate": 1.0924301926274248e-07,
+ "loss": 8.602528396295384e-07,
+ "num_tokens": 1187311.0,
+ "reward": 0.4899469017982483,
+ "reward_std": 0.24049179255962372,
+ "rewards/true_env_reward_fn/mean": 0.4899469017982483,
+ "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "step": 290,
+ "step_time": 4.003069795000556
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 109.0,
+ "completions/max_terminated_length": 109.0,
+ "completions/mean_length": 65.875,
+ "completions/mean_terminated_length": 65.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.222926139831543,
+ "epoch": 2.3658536585365852,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.209110481198877e-05,
+ "kl": 1.1452370472397888e-05,
+ "learning_rate": 1.0665668633705572e-07,
+ "loss": 5.764911747974111e-07,
+ "num_tokens": 1191370.0,
+ "reward": 0.5930472612380981,
+ "reward_std": 0.14818456768989563,
+ "rewards/true_env_reward_fn/mean": 0.5930472612380981,
+ "rewards/true_env_reward_fn/std": 0.14818456768989563,
+ "step": 291,
+ "step_time": 4.77749846199913
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 65.625,
+ "completions/mean_terminated_length": 65.625,
+ "completions/min_length": 56.0,
+ "completions/min_terminated_length": 56.0,
+ "entropy": 1.313057780265808,
+ "epoch": 2.3739837398373984,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.671779556199908e-05,
+ "kl": 1.1384066056052689e-05,
+ "learning_rate": 1.0409661956807174e-07,
+ "loss": 5.689008162335085e-07,
+ "num_tokens": 1195851.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 292,
+ "step_time": 3.4948791150000034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.3050158619880676,
+ "epoch": 2.3821138211382116,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15694627165794373,
+ "kl": 2.6891467314271722e-05,
+ "learning_rate": 1.0156304818588308e-07,
+ "loss": -0.044491954147815704,
+ "num_tokens": 1201316.0,
+ "reward": 0.18382371962070465,
+ "reward_std": 0.18414245545864105,
+ "rewards/true_env_reward_fn/mean": 0.18382371962070465,
+ "rewards/true_env_reward_fn/std": 0.18414245545864105,
+ "step": 293,
+ "step_time": 3.900356202000694
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 52.25,
+ "completions/mean_terminated_length": 52.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2079132199287415,
+ "epoch": 2.3902439024390243,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.25304585695266724,
+ "kl": 4.250183792464668e-05,
+ "learning_rate": 9.905619904816749e-08,
+ "loss": 0.008140146732330322,
+ "num_tokens": 1204090.0,
+ "reward": 0.43626630306243896,
+ "reward_std": 0.3027261197566986,
+ "rewards/true_env_reward_fn/mean": 0.43626630306243896,
+ "rewards/true_env_reward_fn/std": 0.3027261197566986,
+ "step": 294,
+ "step_time": 3.1466946830005327
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2324314713478088,
+ "epoch": 2.3983739837398375,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011851672024931759,
+ "kl": 1.4868088328512385e-05,
+ "learning_rate": 9.657629661987531e-08,
+ "loss": 7.534490578109398e-07,
+ "num_tokens": 1206449.0,
+ "reward": 0.7232838273048401,
+ "reward_std": 0.008955853059887886,
+ "rewards/true_env_reward_fn/mean": 0.7232838273048401,
+ "rewards/true_env_reward_fn/std": 0.008955853059887886,
+ "step": 295,
+ "step_time": 3.5492840760016406
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 65.125,
+ "completions/mean_terminated_length": 65.125,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.2110244631767273,
+ "epoch": 2.40650406504065,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18513579666614532,
+ "kl": 6.467000457632821e-05,
+ "learning_rate": 9.412356295313019e-08,
+ "loss": -0.008508354425430298,
+ "num_tokens": 1211294.0,
+ "reward": 0.38126999139785767,
+ "reward_std": 0.22339044511318207,
+ "rewards/true_env_reward_fn/mean": 0.38126999139785767,
+ "rewards/true_env_reward_fn/std": 0.22339043021202087,
+ "step": 296,
+ "step_time": 3.6129159619995335
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 58.75,
+ "completions/mean_terminated_length": 58.75,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.1828523874282837,
+ "epoch": 2.4146341463414633,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13915051519870758,
+ "kl": 2.76857699645916e-05,
+ "learning_rate": 9.169821766734668e-08,
+ "loss": -0.019797056913375854,
+ "num_tokens": 1214972.0,
+ "reward": 0.5778937339782715,
+ "reward_std": 0.4268997013568878,
+ "rewards/true_env_reward_fn/mean": 0.5778937339782715,
+ "rewards/true_env_reward_fn/std": 0.4268997013568878,
+ "step": 297,
+ "step_time": 3.5299333029997797
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2796242237091064,
+ "epoch": 2.4227642276422765,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23225097358226776,
+ "kl": 5.458398300106637e-05,
+ "learning_rate": 8.930047792956585e-08,
+ "loss": 0.028596192598342896,
+ "num_tokens": 1221117.0,
+ "reward": 0.09695600718259811,
+ "reward_std": 0.23755072057247162,
+ "rewards/true_env_reward_fn/mean": 0.09695600718259811,
+ "rewards/true_env_reward_fn/std": 0.23755072057247162,
+ "step": 298,
+ "step_time": 3.9654863289997593
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.3131609559059143,
+ "epoch": 2.430894308943089,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.15491950511932373,
+ "kl": 2.3678386241954286e-05,
+ "learning_rate": 8.693055843500867e-08,
+ "loss": 0.08264091610908508,
+ "num_tokens": 1226670.0,
+ "reward": 0.22746901214122772,
+ "reward_std": 0.2765822112560272,
+ "rewards/true_env_reward_fn/mean": 0.22746901214122772,
+ "rewards/true_env_reward_fn/std": 0.2765822112560272,
+ "step": 299,
+ "step_time": 3.9300464680000005
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 61.75,
+ "completions/mean_terminated_length": 61.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.113481342792511,
+ "epoch": 2.4390243902439024,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14261895418167114,
+ "kl": 2.8488028874562588e-05,
+ "learning_rate": 8.458867138785369e-08,
+ "loss": 0.012870386242866516,
+ "num_tokens": 1230460.0,
+ "reward": 0.6178936958312988,
+ "reward_std": 0.3830615282058716,
+ "rewards/true_env_reward_fn/mean": 0.6178936958312988,
+ "rewards/true_env_reward_fn/std": 0.3830614984035492,
+ "step": 300,
+ "step_time": 4.004705740000645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 54.25,
+ "completions/mean_terminated_length": 54.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.2633118629455566,
+ "epoch": 2.4471544715447155,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1559160053730011,
+ "kl": 3.0400691230170196e-05,
+ "learning_rate": 8.227502648223494e-08,
+ "loss": 0.023099511861801147,
+ "num_tokens": 1234850.0,
+ "reward": 0.3328326642513275,
+ "reward_std": 0.2606535255908966,
+ "rewards/true_env_reward_fn/mean": 0.3328326642513275,
+ "rewards/true_env_reward_fn/std": 0.260653555393219,
+ "step": 301,
+ "step_time": 4.035395368000536
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 79.875,
+ "completions/mean_terminated_length": 79.875,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4545465111732483,
+ "epoch": 2.4552845528455283,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13911883533000946,
+ "kl": 5.561073703574948e-05,
+ "learning_rate": 7.99898308834662e-08,
+ "loss": 0.10435273498296738,
+ "num_tokens": 1237629.0,
+ "reward": 0.3929310441017151,
+ "reward_std": 0.294041246175766,
+ "rewards/true_env_reward_fn/mean": 0.3929310441017151,
+ "rewards/true_env_reward_fn/std": 0.294041246175766,
+ "step": 302,
+ "step_time": 6.382147416999942
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.4197523593902588,
+ "epoch": 2.4634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11274738609790802,
+ "kl": 1.6755700926296413e-05,
+ "learning_rate": 7.773328920949151e-08,
+ "loss": -0.1040230244398117,
+ "num_tokens": 1240738.0,
+ "reward": 0.6045305728912354,
+ "reward_std": 0.10337947309017181,
+ "rewards/true_env_reward_fn/mean": 0.6045305728912354,
+ "rewards/true_env_reward_fn/std": 0.10337948054075241,
+ "step": 303,
+ "step_time": 3.5754013399982796
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 105.0,
+ "completions/max_terminated_length": 105.0,
+ "completions/mean_length": 69.625,
+ "completions/mean_terminated_length": 69.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.3458821177482605,
+ "epoch": 2.4715447154471546,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11526551097631454,
+ "kl": 2.5664294298621826e-05,
+ "learning_rate": 7.550560351256309e-08,
+ "loss": 0.0017936527729034424,
+ "num_tokens": 1245803.0,
+ "reward": 0.2295326590538025,
+ "reward_std": 0.3014034032821655,
+ "rewards/true_env_reward_fn/mean": 0.2295326590538025,
+ "rewards/true_env_reward_fn/std": 0.3014034032821655,
+ "step": 304,
+ "step_time": 4.8661928239998815
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 93.0,
+ "completions/max_terminated_length": 93.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2587900161743164,
+ "epoch": 2.4796747967479673,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1804642677307129,
+ "kl": 3.644101343525108e-05,
+ "learning_rate": 7.330697326114972e-08,
+ "loss": 0.006622403860092163,
+ "num_tokens": 1250224.0,
+ "reward": 0.392159104347229,
+ "reward_std": 0.23998720943927765,
+ "rewards/true_env_reward_fn/mean": 0.392159104347229,
+ "rewards/true_env_reward_fn/std": 0.23998722434043884,
+ "step": 305,
+ "step_time": 4.381737805999364
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 154.0,
+ "completions/max_terminated_length": 154.0,
+ "completions/mean_length": 88.75,
+ "completions/mean_terminated_length": 88.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.235496699810028,
+ "epoch": 2.4878048780487805,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010392541298642755,
+ "kl": 1.9250182049290743e-05,
+ "learning_rate": 7.113759532207599e-08,
+ "loss": 9.166102472590865e-07,
+ "num_tokens": 1256018.0,
+ "reward": 0.45198333263397217,
+ "reward_std": 0.0026191486977040768,
+ "rewards/true_env_reward_fn/mean": 0.45198333263397217,
+ "rewards/true_env_reward_fn/std": 0.0026191489305347204,
+ "step": 306,
+ "step_time": 6.6232522029986285
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.1592004895210266,
+ "epoch": 2.4959349593495936,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.044789915904403e-05,
+ "kl": 9.38343055167934e-06,
+ "learning_rate": 6.899766394289516e-08,
+ "loss": 4.709004315373022e-07,
+ "num_tokens": 1259980.0,
+ "reward": 0.4911326766014099,
+ "reward_std": 0.01917082816362381,
+ "rewards/true_env_reward_fn/mean": 0.4911326766014099,
+ "rewards/true_env_reward_fn/std": 0.01917083002626896,
+ "step": 307,
+ "step_time": 3.398790989002009
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 66.0,
+ "completions/max_terminated_length": 66.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1962818503379822,
+ "epoch": 2.5040650406504064,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12975044548511505,
+ "kl": 9.622429843147984e-06,
+ "learning_rate": 6.688737073449563e-08,
+ "loss": -0.028225116431713104,
+ "num_tokens": 1265125.0,
+ "reward": 0.3520139753818512,
+ "reward_std": 0.4090423583984375,
+ "rewards/true_env_reward_fn/mean": 0.3520139753818512,
+ "rewards/true_env_reward_fn/std": 0.4090423583984375,
+ "step": 308,
+ "step_time": 3.4128740100004507
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1477364301681519,
+ "epoch": 2.5121951219512195,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.428078242810443e-05,
+ "kl": 1.2252480246388586e-05,
+ "learning_rate": 6.480690465394398e-08,
+ "loss": 6.147511157905683e-07,
+ "num_tokens": 1269679.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 309,
+ "step_time": 3.4761773770005675
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 60.625,
+ "completions/mean_terminated_length": 60.625,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.1964089274406433,
+ "epoch": 2.5203252032520327,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010435594595037401,
+ "kl": 1.349770900560543e-05,
+ "learning_rate": 6.275645198756629e-08,
+ "loss": 6.71000861984794e-07,
+ "num_tokens": 1272836.0,
+ "reward": 0.4899469017982483,
+ "reward_std": 0.24049179255962372,
+ "rewards/true_env_reward_fn/mean": 0.4899469017982483,
+ "rewards/true_env_reward_fn/std": 0.24049179255962372,
+ "step": 310,
+ "step_time": 3.522990450999714
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 399.0,
+ "completions/max_terminated_length": 399.0,
+ "completions/mean_length": 101.125,
+ "completions/mean_terminated_length": 101.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.428375780582428,
+ "epoch": 2.5284552845528454,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.10063836723566055,
+ "kl": 2.3114703708415618e-05,
+ "learning_rate": 6.073619633426675e-08,
+ "loss": -0.1970823109149933,
+ "num_tokens": 1279065.0,
+ "reward": 0.4598180055618286,
+ "reward_std": 0.23299095034599304,
+ "rewards/true_env_reward_fn/mean": 0.4598180055618286,
+ "rewards/true_env_reward_fn/std": 0.23299095034599304,
+ "step": 311,
+ "step_time": 16.64716850199875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 62.875,
+ "completions/mean_terminated_length": 62.875,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0843100249767303,
+ "epoch": 2.5365853658536586,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00015719492512289435,
+ "kl": 1.4557146187144099e-05,
+ "learning_rate": 5.8746318589089337e-08,
+ "loss": 7.280061709025176e-07,
+ "num_tokens": 1280592.0,
+ "reward": 0.8541955947875977,
+ "reward_std": 0.09160846471786499,
+ "rewards/true_env_reward_fn/mean": 0.8541955947875977,
+ "rewards/true_env_reward_fn/std": 0.09160846471786499,
+ "step": 312,
+ "step_time": 3.2888442910007143
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 72.0,
+ "completions/mean_terminated_length": 72.0,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.240959882736206,
+ "epoch": 2.5447154471544717,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12602119147777557,
+ "kl": 1.7604435015527997e-05,
+ "learning_rate": 5.67869969270196e-08,
+ "loss": 9.685754776000977e-07,
+ "num_tokens": 1284772.0,
+ "reward": 0.4882529377937317,
+ "reward_std": 0.2615475356578827,
+ "rewards/true_env_reward_fn/mean": 0.4882529377937317,
+ "rewards/true_env_reward_fn/std": 0.2615475356578827,
+ "step": 313,
+ "step_time": 4.0729800409990276
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1718184351921082,
+ "epoch": 2.5528455284552845,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14716969430446625,
+ "kl": 1.3087485967844259e-05,
+ "learning_rate": 5.4858406787030845e-08,
+ "loss": 0.016231566667556763,
+ "num_tokens": 1287839.0,
+ "reward": 0.6037359237670898,
+ "reward_std": 0.10266375541687012,
+ "rewards/true_env_reward_fn/mean": 0.6037359237670898,
+ "rewards/true_env_reward_fn/std": 0.10266375541687012,
+ "step": 314,
+ "step_time": 3.175157601999672
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 52.0,
+ "completions/max_terminated_length": 52.0,
+ "completions/mean_length": 40.5,
+ "completions/mean_terminated_length": 40.5,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.3117390871047974,
+ "epoch": 2.5609756097560976,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2794886827468872,
+ "kl": 4.65317443740787e-05,
+ "learning_rate": 5.29607208563756e-08,
+ "loss": -0.09303665161132812,
+ "num_tokens": 1291515.0,
+ "reward": 0.5445280075073242,
+ "reward_std": 0.19415061175823212,
+ "rewards/true_env_reward_fn/mean": 0.5445280075073242,
+ "rewards/true_env_reward_fn/std": 0.19415059685707092,
+ "step": 315,
+ "step_time": 2.636586960999921
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
+ "completions/min_length": 13.0,
+ "completions/min_terminated_length": 13.0,
+ "entropy": 1.15413236618042,
+ "epoch": 2.569105691056911,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22290591895580292,
+ "kl": 6.188569841469871e-05,
+ "learning_rate": 5.10941090551225e-08,
+ "loss": -0.013920806348323822,
+ "num_tokens": 1296399.0,
+ "reward": 0.3418610095977783,
+ "reward_std": 0.29834023118019104,
+ "rewards/true_env_reward_fn/mean": 0.3418610095977783,
+ "rewards/true_env_reward_fn/std": 0.2983402609825134,
+ "step": 316,
+ "step_time": 3.7106533750011295
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 265.0,
+ "completions/max_terminated_length": 265.0,
+ "completions/mean_length": 102.875,
+ "completions/mean_terminated_length": 102.875,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.2713149785995483,
+ "epoch": 2.5772357723577235,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11319029331207275,
+ "kl": 1.3128932096151402e-05,
+ "learning_rate": 4.9258738520942025e-08,
+ "loss": 0.04848237335681915,
+ "num_tokens": 1301626.0,
+ "reward": 0.4802166819572449,
+ "reward_std": 0.05550921708345413,
+ "rewards/true_env_reward_fn/mean": 0.4802166819572449,
+ "rewards/true_env_reward_fn/std": 0.05550922453403473,
+ "step": 317,
+ "step_time": 11.140286670997739
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 64.0,
+ "completions/max_terminated_length": 64.0,
+ "completions/mean_length": 45.625,
+ "completions/mean_terminated_length": 45.625,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 0.9857950508594513,
+ "epoch": 2.5853658536585367,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011810586147475988,
+ "kl": 1.2108953342249151e-05,
+ "learning_rate": 4.745477359414045e-08,
+ "loss": 6.04832052886195e-07,
+ "num_tokens": 1303619.0,
+ "reward": 0.6204532384872437,
+ "reward_std": 0.10097470134496689,
+ "rewards/true_env_reward_fn/mean": 0.6204532384872437,
+ "rewards/true_env_reward_fn/std": 0.10097470879554749,
+ "step": 318,
+ "step_time": 2.977002043000539
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 53.875,
+ "completions/mean_terminated_length": 53.875,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2560631036758423,
+ "epoch": 2.59349593495935,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.24155104160308838,
+ "kl": 3.4935148505610414e-05,
+ "learning_rate": 4.5682375802945295e-08,
+ "loss": 0.1817030906677246,
+ "num_tokens": 1310682.0,
+ "reward": 0.055160000920295715,
+ "reward_std": 0.13755998015403748,
+ "rewards/true_env_reward_fn/mean": 0.055160000920295715,
+ "rewards/true_env_reward_fn/std": 0.13755998015403748,
+ "step": 319,
+ "step_time": 4.712957282001298
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 102.0,
+ "completions/max_terminated_length": 102.0,
+ "completions/mean_length": 69.75,
+ "completions/mean_terminated_length": 69.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2313570380210876,
+ "epoch": 2.6016260162601625,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011636545968940482,
+ "kl": 1.566714945511194e-05,
+ "learning_rate": 4.394170384904137e-08,
+ "loss": 7.867492968216538e-07,
+ "num_tokens": 1315436.0,
+ "reward": 0.4662666618824005,
+ "reward_std": 0.007412042934447527,
+ "rewards/true_env_reward_fn/mean": 0.4662666618824005,
+ "rewards/true_env_reward_fn/std": 0.007412043400108814,
+ "step": 320,
+ "step_time": 4.536037327998201
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 49.625,
+ "completions/mean_terminated_length": 49.625,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.0671695470809937,
+ "epoch": 2.6097560975609757,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00013134608161635697,
+ "kl": 1.7747691344993655e-05,
+ "learning_rate": 4.223291359336074e-08,
+ "loss": 8.864202527547604e-07,
+ "num_tokens": 1318837.0,
+ "reward": 0.6651140451431274,
+ "reward_std": 0.20516473054885864,
+ "rewards/true_env_reward_fn/mean": 0.6651140451431274,
+ "rewards/true_env_reward_fn/std": 0.20516474545001984,
+ "step": 321,
+ "step_time": 2.9819667609990574
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 178.0,
+ "completions/max_terminated_length": 178.0,
+ "completions/mean_length": 85.25,
+ "completions/mean_terminated_length": 85.25,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3910706639289856,
+ "epoch": 2.617886178861789,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1727662831544876,
+ "kl": 1.9147763850924093e-05,
+ "learning_rate": 4.05561580421272e-08,
+ "loss": 0.07752666622400284,
+ "num_tokens": 1324515.0,
+ "reward": 0.43344998359680176,
+ "reward_std": 0.039401449263095856,
+ "rewards/true_env_reward_fn/mean": 0.43344998359680176,
+ "rewards/true_env_reward_fn/std": 0.039401449263095856,
+ "step": 322,
+ "step_time": 7.645187717998851
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 92.0,
+ "completions/max_terminated_length": 92.0,
+ "completions/mean_length": 65.125,
+ "completions/mean_terminated_length": 65.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.1752581596374512,
+ "epoch": 2.6260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14819224178791046,
+ "kl": 1.1951405213039834e-05,
+ "learning_rate": 3.891158733315505e-08,
+ "loss": 0.0940035805106163,
+ "num_tokens": 1327268.0,
+ "reward": 0.6361198425292969,
+ "reward_std": 0.13543039560317993,
+ "rewards/true_env_reward_fn/mean": 0.6361198425292969,
+ "rewards/true_env_reward_fn/std": 0.13543038070201874,
+ "step": 323,
+ "step_time": 4.102753150000353
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.75,
+ "completions/mean_terminated_length": 69.75,
+ "completions/min_length": 29.0,
+ "completions/min_terminated_length": 29.0,
+ "entropy": 1.4734079241752625,
+ "epoch": 2.6341463414634148,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21136029064655304,
+ "kl": 4.428037027537357e-05,
+ "learning_rate": 3.7299348722406965e-08,
+ "loss": -0.13074398040771484,
+ "num_tokens": 1331614.0,
+ "reward": 0.3051303029060364,
+ "reward_std": 0.20326493680477142,
+ "rewards/true_env_reward_fn/mean": 0.3051303029060364,
+ "rewards/true_env_reward_fn/std": 0.20326495170593262,
+ "step": 324,
+ "step_time": 5.413495869999679
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 179.0,
+ "completions/max_terminated_length": 179.0,
+ "completions/mean_length": 78.625,
+ "completions/mean_terminated_length": 78.625,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.5631850361824036,
+ "epoch": 2.642276422764228,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18741382658481598,
+ "kl": 3.721813300217036e-05,
+ "learning_rate": 3.57195865708074e-08,
+ "loss": 0.20410414040088654,
+ "num_tokens": 1334991.0,
+ "reward": 0.4402502775192261,
+ "reward_std": 0.27313899993896484,
+ "rewards/true_env_reward_fn/mean": 0.4402502775192261,
+ "rewards/true_env_reward_fn/std": 0.27313899993896484,
+ "step": 325,
+ "step_time": 7.439067015000546
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2137504816055298,
+ "epoch": 2.6504065040650406,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1392536610364914,
+ "kl": 2.5514624212519266e-05,
+ "learning_rate": 3.417244233131709e-08,
+ "loss": -0.07728786766529083,
+ "num_tokens": 1340886.0,
+ "reward": 0.3032499849796295,
+ "reward_std": 0.24293993413448334,
+ "rewards/true_env_reward_fn/mean": 0.3032499849796295,
+ "rewards/true_env_reward_fn/std": 0.24293994903564453,
+ "step": 326,
+ "step_time": 3.6837621579998086
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 45.125,
+ "completions/mean_terminated_length": 45.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1947113871574402,
+ "epoch": 2.658536585365854,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23312504589557648,
+ "kl": 6.829304766142741e-05,
+ "learning_rate": 3.265805453626713e-08,
+ "loss": 0.08558641374111176,
+ "num_tokens": 1344823.0,
+ "reward": 0.26869305968284607,
+ "reward_std": 0.30387282371520996,
+ "rewards/true_env_reward_fn/mean": 0.26869305968284607,
+ "rewards/true_env_reward_fn/std": 0.30387285351753235,
+ "step": 327,
+ "step_time": 3.26592919200084
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 70.125,
+ "completions/mean_terminated_length": 70.125,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.1701073944568634,
+ "epoch": 2.6666666666666665,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12238985300064087,
+ "kl": 1.6267454157059547e-05,
+ "learning_rate": 3.117655878495458e-08,
+ "loss": -0.024053269997239113,
+ "num_tokens": 1348012.0,
+ "reward": 0.5990839004516602,
+ "reward_std": 0.21510076522827148,
+ "rewards/true_env_reward_fn/mean": 0.5990839004516602,
+ "rewards/true_env_reward_fn/std": 0.21510076522827148,
+ "step": 328,
+ "step_time": 4.166060536001169
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 61.0,
+ "completions/max_terminated_length": 61.0,
+ "completions/mean_length": 49.75,
+ "completions/mean_terminated_length": 49.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.3588789701461792,
+ "epoch": 2.6747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21684816479682922,
+ "kl": 0.00012067620264133438,
+ "learning_rate": 2.9728087731500884e-08,
+ "loss": 0.017311420291662216,
+ "num_tokens": 1352082.0,
+ "reward": 0.15680472552776337,
+ "reward_std": 0.3331984877586365,
+ "rewards/true_env_reward_fn/mean": 0.15680472552776337,
+ "rewards/true_env_reward_fn/std": 0.3331984877586365,
+ "step": 329,
+ "step_time": 2.947034541000903
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 57.125,
+ "completions/mean_terminated_length": 57.125,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.2207121849060059,
+ "epoch": 2.682926829268293,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14501330256462097,
+ "kl": 3.47198892995948e-05,
+ "learning_rate": 2.831277107297394e-08,
+ "loss": -0.009820610284805298,
+ "num_tokens": 1356867.0,
+ "reward": 0.508467972278595,
+ "reward_std": 0.10515833646059036,
+ "rewards/true_env_reward_fn/mean": 0.508467972278595,
+ "rewards/true_env_reward_fn/std": 0.10515833646059036,
+ "step": 330,
+ "step_time": 3.278030399000272
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 121.0,
+ "completions/max_terminated_length": 121.0,
+ "completions/mean_length": 71.5,
+ "completions/mean_terminated_length": 71.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2053698897361755,
+ "epoch": 2.6910569105691056,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1709476113319397,
+ "kl": 1.9107792468275875e-05,
+ "learning_rate": 2.693073553777494e-08,
+ "loss": 0.04886455833911896,
+ "num_tokens": 1361747.0,
+ "reward": 0.3092518746852875,
+ "reward_std": 0.40765801072120667,
+ "rewards/true_env_reward_fn/mean": 0.3092518746852875,
+ "rewards/true_env_reward_fn/std": 0.40765801072120667,
+ "step": 331,
+ "step_time": 5.705391269999382
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 70.625,
+ "completions/mean_terminated_length": 70.625,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.3019116520881653,
+ "epoch": 2.6991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13118988275527954,
+ "kl": 6.723498518113047e-05,
+ "learning_rate": 2.5582104874290888e-08,
+ "loss": 0.04069610685110092,
+ "num_tokens": 1366792.0,
+ "reward": 0.39014729857444763,
+ "reward_std": 0.4048542380332947,
+ "rewards/true_env_reward_fn/mean": 0.39014729857444763,
+ "rewards/true_env_reward_fn/std": 0.40485426783561707,
+ "step": 332,
+ "step_time": 4.941442722001739
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.125,
+ "completions/max_length": 512.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 122.25,
+ "completions/mean_terminated_length": 66.5714340209961,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 0.8574695587158203,
+ "epoch": 2.7073170731707314,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.08299780637025833,
+ "kl": 8.204095138353296e-06,
+ "learning_rate": 2.4266999839814484e-08,
+ "loss": -0.5517755150794983,
+ "num_tokens": 1373066.0,
+ "reward": 0.19574999809265137,
+ "reward_std": 0.30308666825294495,
+ "rewards/true_env_reward_fn/mean": 0.19574999809265137,
+ "rewards/true_env_reward_fn/std": 0.30308666825294495,
+ "step": 333,
+ "step_time": 20.7688721859995
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 49.625,
+ "completions/mean_terminated_length": 49.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.191849946975708,
+ "epoch": 2.7154471544715446,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001034519009408541,
+ "kl": 1.4263560842664447e-05,
+ "learning_rate": 2.2985538189730946e-08,
+ "loss": 7.244366315717343e-07,
+ "num_tokens": 1375815.0,
+ "reward": 0.6581807136535645,
+ "reward_std": 0.21257677674293518,
+ "rewards/true_env_reward_fn/mean": 0.6581807136535645,
+ "rewards/true_env_reward_fn/std": 0.21257677674293518,
+ "step": 334,
+ "step_time": 3.5656982629971026
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 59.75,
+ "completions/mean_terminated_length": 59.75,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.2259244322776794,
+ "epoch": 2.7235772357723578,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12885747849941254,
+ "kl": 4.455958514881786e-05,
+ "learning_rate": 2.1737834666974276e-08,
+ "loss": -0.06481382250785828,
+ "num_tokens": 1379933.0,
+ "reward": 0.31903696060180664,
+ "reward_std": 0.1877410113811493,
+ "rewards/true_env_reward_fn/mean": 0.31903696060180664,
+ "rewards/true_env_reward_fn/std": 0.18774102628231049,
+ "step": 335,
+ "step_time": 3.712153099000716
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 63.0,
+ "completions/max_terminated_length": 63.0,
+ "completions/mean_length": 45.0,
+ "completions/mean_terminated_length": 45.0,
+ "completions/min_length": 26.0,
+ "completions/min_terminated_length": 26.0,
+ "entropy": 1.2184038758277893,
+ "epoch": 2.7317073170731705,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.258829265832901,
+ "kl": 6.149329965410288e-05,
+ "learning_rate": 2.0524000991753465e-08,
+ "loss": -0.04611395671963692,
+ "num_tokens": 1383205.0,
+ "reward": 0.41517797112464905,
+ "reward_std": 0.33201757073402405,
+ "rewards/true_env_reward_fn/mean": 0.41517797112464905,
+ "rewards/true_env_reward_fn/std": 0.33201757073402405,
+ "step": 336,
+ "step_time": 3.02953305399933
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 206.0,
+ "completions/max_terminated_length": 206.0,
+ "completions/mean_length": 87.875,
+ "completions/mean_terminated_length": 87.875,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.2387716174125671,
+ "epoch": 2.7398373983739837,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16021011769771576,
+ "kl": 1.6921320366236614e-05,
+ "learning_rate": 1.9344145851548243e-08,
+ "loss": -0.14579075574874878,
+ "num_tokens": 1389136.0,
+ "reward": 0.4916413426399231,
+ "reward_std": 0.22573381662368774,
+ "rewards/true_env_reward_fn/mean": 0.4916413426399231,
+ "rewards/true_env_reward_fn/std": 0.22573381662368774,
+ "step": 337,
+ "step_time": 9.208789625999998
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 50.75,
+ "completions/mean_terminated_length": 50.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.2942972779273987,
+ "epoch": 2.747967479674797,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2804953157901764,
+ "kl": 6.65090201437124e-05,
+ "learning_rate": 1.8198374891377966e-08,
+ "loss": 0.14034238457679749,
+ "num_tokens": 1392442.0,
+ "reward": 0.458748459815979,
+ "reward_std": 0.24402853846549988,
+ "rewards/true_env_reward_fn/mean": 0.458748459815979,
+ "rewards/true_env_reward_fn/std": 0.24402853846549988,
+ "step": 338,
+ "step_time": 3.383874344001015
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 99.0,
+ "completions/max_terminated_length": 99.0,
+ "completions/mean_length": 75.625,
+ "completions/mean_terminated_length": 75.625,
+ "completions/min_length": 60.0,
+ "completions/min_terminated_length": 60.0,
+ "entropy": 1.433261752128601,
+ "epoch": 2.7560975609756095,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11875848472118378,
+ "kl": 2.3840807443775702e-05,
+ "learning_rate": 1.7086790704341402e-08,
+ "loss": -0.015734028071165085,
+ "num_tokens": 1396235.0,
+ "reward": 0.3571140468120575,
+ "reward_std": 0.10167346149682999,
+ "rewards/true_env_reward_fn/mean": 0.3571140468120575,
+ "rewards/true_env_reward_fn/std": 0.10167345404624939,
+ "step": 339,
+ "step_time": 4.3859739109993825
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 186.0,
+ "completions/max_terminated_length": 186.0,
+ "completions/mean_length": 97.75,
+ "completions/mean_terminated_length": 97.75,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.5857495069503784,
+ "epoch": 2.7642276422764227,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 6.698282959405333e-05,
+ "kl": 1.4666956758446759e-05,
+ "learning_rate": 1.600949282243089e-08,
+ "loss": 7.313840342249023e-07,
+ "num_tokens": 1401073.0,
+ "reward": 0.3621603846549988,
+ "reward_std": 0.10388225317001343,
+ "rewards/true_env_reward_fn/mean": 0.3621603846549988,
+ "rewards/true_env_reward_fn/std": 0.10388225317001343,
+ "step": 340,
+ "step_time": 7.751754675999109
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 54.0,
+ "completions/mean_terminated_length": 54.0,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.0097725987434387,
+ "epoch": 2.772357723577236,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011211638775421306,
+ "kl": 1.3342013517103624e-05,
+ "learning_rate": 1.496657770762013e-08,
+ "loss": 6.635225417994661e-07,
+ "num_tokens": 1403921.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 341,
+ "step_time": 3.4684136910000234
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.1732905507087708,
+ "epoch": 2.7804878048780486,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14162735641002655,
+ "kl": 4.145995080762077e-05,
+ "learning_rate": 1.3958138743226688e-08,
+ "loss": 0.007000558078289032,
+ "num_tokens": 1407681.0,
+ "reward": 0.5386021733283997,
+ "reward_std": 0.31964072585105896,
+ "rewards/true_env_reward_fn/mean": 0.5386021733283997,
+ "rewards/true_env_reward_fn/std": 0.31964072585105896,
+ "step": 342,
+ "step_time": 3.5975158429992007
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 67.625,
+ "completions/mean_terminated_length": 67.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0845041275024414,
+ "epoch": 2.7886178861788617,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.0979820266366005,
+ "kl": 1.916076598718064e-05,
+ "learning_rate": 1.29842662255506e-08,
+ "loss": -0.049884289503097534,
+ "num_tokens": 1413182.0,
+ "reward": 0.316629558801651,
+ "reward_std": 0.16504530608654022,
+ "rewards/true_env_reward_fn/mean": 0.316629558801651,
+ "rewards/true_env_reward_fn/std": 0.16504532098770142,
+ "step": 343,
+ "step_time": 4.6961259659983625
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 51.25,
+ "completions/mean_terminated_length": 51.25,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.1820043921470642,
+ "epoch": 2.796747967479675,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1726708859205246,
+ "kl": 2.1745819140051026e-05,
+ "learning_rate": 1.2045047355789107e-08,
+ "loss": 0.048721786588430405,
+ "num_tokens": 1415472.0,
+ "reward": 0.6344382762908936,
+ "reward_std": 0.07378704100847244,
+ "rewards/true_env_reward_fn/mean": 0.6344382762908936,
+ "rewards/true_env_reward_fn/std": 0.07378704845905304,
+ "step": 344,
+ "step_time": 3.2691068980002456
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 59.625,
+ "completions/mean_terminated_length": 59.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.0933732390403748,
+ "epoch": 2.8048780487804876,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21739810705184937,
+ "kl": 2.0197898265905678e-05,
+ "learning_rate": 1.1140566232228677e-08,
+ "loss": 0.0031427331268787384,
+ "num_tokens": 1417385.0,
+ "reward": 0.5309104323387146,
+ "reward_std": 0.18906596302986145,
+ "rewards/true_env_reward_fn/mean": 0.5309104323387146,
+ "rewards/true_env_reward_fn/std": 0.18906594812870026,
+ "step": 345,
+ "step_time": 3.9515345389991126
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 58.875,
+ "completions/mean_terminated_length": 58.875,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.2968629598617554,
+ "epoch": 2.813008130081301,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20110434293746948,
+ "kl": 4.813342457055114e-05,
+ "learning_rate": 1.0270903842714673e-08,
+ "loss": 0.062390223145484924,
+ "num_tokens": 1423496.0,
+ "reward": 0.14946666359901428,
+ "reward_std": 0.24863645434379578,
+ "rewards/true_env_reward_fn/mean": 0.14946666359901428,
+ "rewards/true_env_reward_fn/std": 0.24863646924495697,
+ "step": 346,
+ "step_time": 4.331638548001138
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 63.75,
+ "completions/mean_terminated_length": 63.75,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.153487741947174,
+ "epoch": 2.821138211382114,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00011695526336552575,
+ "kl": 1.7154279703390785e-05,
+ "learning_rate": 9.436138057399601e-09,
+ "loss": 8.606326673543663e-07,
+ "num_tokens": 1427182.0,
+ "reward": 0.5776733160018921,
+ "reward_std": 0.11168669909238815,
+ "rewards/true_env_reward_fn/mean": 0.5776733160018921,
+ "rewards/true_env_reward_fn/std": 0.11168669164180756,
+ "step": 347,
+ "step_time": 3.416529175998221
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 61.125,
+ "completions/mean_terminated_length": 61.125,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3210760354995728,
+ "epoch": 2.8292682926829267,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13752248883247375,
+ "kl": 3.656550507002976e-05,
+ "learning_rate": 8.636343621770948e-09,
+ "loss": -0.029253121465444565,
+ "num_tokens": 1431839.0,
+ "reward": 0.47378063201904297,
+ "reward_std": 0.32937321066856384,
+ "rewards/true_env_reward_fn/mean": 0.47378063201904297,
+ "rewards/true_env_reward_fn/std": 0.32937324047088623,
+ "step": 348,
+ "step_time": 3.957806388998506
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 129.0,
+ "completions/max_terminated_length": 129.0,
+ "completions/mean_length": 83.75,
+ "completions/mean_terminated_length": 83.75,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.3464251160621643,
+ "epoch": 2.83739837398374,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11163587868213654,
+ "kl": 1.8057865418086294e-05,
+ "learning_rate": 7.87159214995774e-09,
+ "loss": 0.01491672545671463,
+ "num_tokens": 1434785.0,
+ "reward": 0.6471105217933655,
+ "reward_std": 0.1651550978422165,
+ "rewards/true_env_reward_fn/mean": 0.6471105217933655,
+ "rewards/true_env_reward_fn/std": 0.1651550978422165,
+ "step": 349,
+ "step_time": 5.513293031999638
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.0,
+ "completions/mean_terminated_length": 57.0,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.1192307472229004,
+ "epoch": 2.845528455284553,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20383992791175842,
+ "kl": 1.563574414831237e-05,
+ "learning_rate": 7.141952118318961e-09,
+ "loss": 0.113973468542099,
+ "num_tokens": 1437333.0,
+ "reward": 0.5760529637336731,
+ "reward_std": 0.11078595370054245,
+ "rewards/true_env_reward_fn/mean": 0.5760529637336731,
+ "rewards/true_env_reward_fn/std": 0.11078596115112305,
+ "step": 350,
+ "step_time": 3.624226386000373
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 60.0,
+ "completions/max_terminated_length": 60.0,
+ "completions/mean_length": 50.375,
+ "completions/mean_terminated_length": 50.375,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.0234483480453491,
+ "epoch": 2.8536585365853657,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22902871668338776,
+ "kl": 1.8781988728733268e-05,
+ "learning_rate": 6.447488859311434e-09,
+ "loss": 0.057678595185279846,
+ "num_tokens": 1442340.0,
+ "reward": 0.198770672082901,
+ "reward_std": 0.2336467057466507,
+ "rewards/true_env_reward_fn/mean": 0.198770672082901,
+ "rewards/true_env_reward_fn/std": 0.2336467206478119,
+ "step": 351,
+ "step_time": 3.130611584001599
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 80.0,
+ "completions/mean_terminated_length": 80.0,
+ "completions/min_length": 61.0,
+ "completions/min_terminated_length": 61.0,
+ "entropy": 1.3876641988754272,
+ "epoch": 2.861788617886179,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010214409849140793,
+ "kl": 1.5831786186026875e-05,
+ "learning_rate": 5.788264555640321e-09,
+ "loss": 7.920572215880384e-07,
+ "num_tokens": 1446264.0,
+ "reward": 0.5599601864814758,
+ "reward_std": 0.18355616927146912,
+ "rewards/true_env_reward_fn/mean": 0.5599601864814758,
+ "rewards/true_env_reward_fn/std": 0.18355616927146912,
+ "step": 352,
+ "step_time": 4.876645256999836
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 58.0,
+ "completions/max_terminated_length": 58.0,
+ "completions/mean_length": 39.25,
+ "completions/mean_terminated_length": 39.25,
+ "completions/min_length": 28.0,
+ "completions/min_terminated_length": 28.0,
+ "entropy": 1.1021025776863098,
+ "epoch": 2.869918699186992,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2626732885837555,
+ "kl": 8.65587744556251e-05,
+ "learning_rate": 5.164338234691223e-09,
+ "loss": -0.008459150791168213,
+ "num_tokens": 1448502.0,
+ "reward": 0.4228404760360718,
+ "reward_std": 0.2365853488445282,
+ "rewards/true_env_reward_fn/mean": 0.4228404760360718,
+ "rewards/true_env_reward_fn/std": 0.2365853488445282,
+ "step": 353,
+ "step_time": 2.757270309000887
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 69.875,
+ "completions/mean_terminated_length": 69.875,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.2293291091918945,
+ "epoch": 2.8780487804878048,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.16841711103916168,
+ "kl": 5.113930455991067e-05,
+ "learning_rate": 4.575765763244588e-09,
+ "loss": 0.04984408989548683,
+ "num_tokens": 1452689.0,
+ "reward": 0.40843212604522705,
+ "reward_std": 0.3593071401119232,
+ "rewards/true_env_reward_fn/mean": 0.40843212604522705,
+ "rewards/true_env_reward_fn/std": 0.3593071401119232,
+ "step": 354,
+ "step_time": 5.442687009999645
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 125.0,
+ "completions/max_terminated_length": 125.0,
+ "completions/mean_length": 74.625,
+ "completions/mean_terminated_length": 74.625,
+ "completions/min_length": 51.0,
+ "completions/min_terminated_length": 51.0,
+ "entropy": 1.0640217959880829,
+ "epoch": 2.886178861788618,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.136996790766716,
+ "kl": 3.551322924977285e-05,
+ "learning_rate": 4.022599842473573e-09,
+ "loss": 0.11179788410663605,
+ "num_tokens": 1456614.0,
+ "reward": 0.4338162839412689,
+ "reward_std": 0.2370380014181137,
+ "rewards/true_env_reward_fn/mean": 0.4338162839412689,
+ "rewards/true_env_reward_fn/std": 0.2370380163192749,
+ "step": 355,
+ "step_time": 5.399883596001018
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 71.0,
+ "completions/max_terminated_length": 71.0,
+ "completions/mean_length": 59.25,
+ "completions/mean_terminated_length": 59.25,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.0311343371868134,
+ "epoch": 2.894308943089431,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.350212738150731e-05,
+ "kl": 1.1659412393782986e-05,
+ "learning_rate": 3.5048900032250696e-09,
+ "loss": 5.831132625644386e-07,
+ "num_tokens": 1458100.0,
+ "reward": 0.9398874640464783,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.9398874640464783,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 356,
+ "step_time": 3.227959533000103
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 64.25,
+ "completions/mean_terminated_length": 64.25,
+ "completions/min_length": 57.0,
+ "completions/min_terminated_length": 57.0,
+ "entropy": 1.3024067282676697,
+ "epoch": 2.902439024390244,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.00010034588194685057,
+ "kl": 1.323698234045878e-05,
+ "learning_rate": 3.0226826015846698e-09,
+ "loss": 6.631016731262207e-07,
+ "num_tokens": 1461586.0,
+ "reward": 0.4486622214317322,
+ "reward_std": 0.06457363814115524,
+ "rewards/true_env_reward_fn/mean": 0.4486622214317322,
+ "rewards/true_env_reward_fn/std": 0.06457363814115524,
+ "step": 357,
+ "step_time": 3.207970530998864
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 63.375,
+ "completions/mean_terminated_length": 63.375,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1890360713005066,
+ "epoch": 2.910569105691057,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.47556148073636e-05,
+ "kl": 1.2006465567537816e-05,
+ "learning_rate": 2.576020814726032e-09,
+ "loss": 6.021950866852421e-07,
+ "num_tokens": 1464777.0,
+ "reward": 0.7316612601280212,
+ "reward_std": 0.0,
+ "rewards/true_env_reward_fn/mean": 0.7316612601280212,
+ "rewards/true_env_reward_fn/std": 0.0,
+ "step": 358,
+ "step_time": 3.9381501269999717
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 60.0,
+ "completions/max_terminated_length": 60.0,
+ "completions/mean_length": 44.375,
+ "completions/mean_terminated_length": 44.375,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.2080550789833069,
+ "epoch": 2.91869918699187,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2057954967021942,
+ "kl": 3.289581036369782e-05,
+ "learning_rate": 2.1649446370445525e-09,
+ "loss": 0.02233278751373291,
+ "num_tokens": 1468752.0,
+ "reward": 0.2706890106201172,
+ "reward_std": 0.22834433615207672,
+ "rewards/true_env_reward_fn/mean": 0.2706890106201172,
+ "rewards/true_env_reward_fn/std": 0.22834433615207672,
+ "step": 359,
+ "step_time": 2.974299924000661
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 70.375,
+ "completions/mean_terminated_length": 70.375,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2017377018928528,
+ "epoch": 2.926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11847318708896637,
+ "kl": 2.8918681891809683e-05,
+ "learning_rate": 1.7894908765764494e-09,
+ "loss": 1.2603788945853012e-06,
+ "num_tokens": 1473959.0,
+ "reward": 0.3668679893016815,
+ "reward_std": 0.29886728525161743,
+ "rewards/true_env_reward_fn/mean": 0.3668679893016815,
+ "rewards/true_env_reward_fn/std": 0.29886728525161743,
+ "step": 360,
+ "step_time": 4.137127149999287
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 41.0,
+ "completions/mean_terminated_length": 41.0,
+ "completions/min_length": 30.0,
+ "completions/min_terminated_length": 30.0,
+ "entropy": 1.3280633687973022,
+ "epoch": 2.934959349593496,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.26977241039276123,
+ "kl": 4.886753777100239e-05,
+ "learning_rate": 1.449693151702691e-09,
+ "loss": 0.07037001848220825,
+ "num_tokens": 1479531.0,
+ "reward": 0.1491166651248932,
+ "reward_std": 0.2235233038663864,
+ "rewards/true_env_reward_fn/mean": 0.1491166651248932,
+ "rewards/true_env_reward_fn/std": 0.2235233187675476,
+ "step": 361,
+ "step_time": 3.1497742850006034
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.3032925724983215,
+ "epoch": 2.943089430894309,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1144510880112648,
+ "kl": 1.66988797900558e-05,
+ "learning_rate": 1.14558188813918e-09,
+ "loss": 0.03772354871034622,
+ "num_tokens": 1483220.0,
+ "reward": 0.5760530233383179,
+ "reward_std": 0.041637521237134933,
+ "rewards/true_env_reward_fn/mean": 0.5760530233383179,
+ "rewards/true_env_reward_fn/std": 0.041637521237134933,
+ "step": 362,
+ "step_time": 4.179528040998775
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.0,
+ "completions/mean_terminated_length": 66.0,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.2688179016113281,
+ "epoch": 2.951219512195122,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.12362973392009735,
+ "kl": 1.415179258401622e-05,
+ "learning_rate": 8.771843162120429e-10,
+ "loss": 0.01135706901550293,
+ "num_tokens": 1487104.0,
+ "reward": 0.4925253391265869,
+ "reward_std": 0.10813789069652557,
+ "rewards/true_env_reward_fn/mean": 0.4925253391265869,
+ "rewards/true_env_reward_fn/std": 0.10813789069652557,
+ "step": 363,
+ "step_time": 3.926700363999771
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 68.625,
+ "completions/mean_terminated_length": 68.625,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.2125371098518372,
+ "epoch": 2.959349593495935,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1332135945558548,
+ "kl": 9.927162409439916e-06,
+ "learning_rate": 6.445244684194939e-10,
+ "loss": -0.012601003050804138,
+ "num_tokens": 1491513.0,
+ "reward": 0.3142518699169159,
+ "reward_std": 0.48772916197776794,
+ "rewards/true_env_reward_fn/mean": 0.3142518699169159,
+ "rewards/true_env_reward_fn/std": 0.48772916197776794,
+ "step": 364,
+ "step_time": 4.090762199999517
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 80.0,
+ "completions/max_terminated_length": 80.0,
+ "completions/mean_length": 64.875,
+ "completions/mean_terminated_length": 64.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.1699539721012115,
+ "epoch": 2.9674796747967482,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17957495152950287,
+ "kl": 3.784914770221803e-05,
+ "learning_rate": 4.4762317728013197e-10,
+ "loss": -0.002692282199859619,
+ "num_tokens": 1496596.0,
+ "reward": 0.05613332986831665,
+ "reward_std": 0.22318392992019653,
+ "rewards/true_env_reward_fn/mean": 0.05613332986831665,
+ "rewards/true_env_reward_fn/std": 0.22318394482135773,
+ "step": 365,
+ "step_time": 3.826017782001145
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 132.0,
+ "completions/max_terminated_length": 132.0,
+ "completions/mean_length": 79.75,
+ "completions/mean_terminated_length": 79.75,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3995569944381714,
+ "epoch": 2.975609756097561,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11512420326471329,
+ "kl": 2.4585679966548923e-05,
+ "learning_rate": 2.864980734674116e-10,
+ "loss": 0.1089683324098587,
+ "num_tokens": 1502482.0,
+ "reward": 0.2563166618347168,
+ "reward_std": 0.25294405221939087,
+ "rewards/true_env_reward_fn/mean": 0.2563166618347168,
+ "rewards/true_env_reward_fn/std": 0.2529440224170685,
+ "step": 366,
+ "step_time": 6.000419455998781
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 65.375,
+ "completions/mean_terminated_length": 65.375,
+ "completions/min_length": 50.0,
+ "completions/min_terminated_length": 50.0,
+ "entropy": 1.3433619141578674,
+ "epoch": 2.983739837398374,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.1911962628364563,
+ "kl": 4.962808270647656e-05,
+ "learning_rate": 1.6116358423099442e-10,
+ "loss": 0.041197627782821655,
+ "num_tokens": 1506793.0,
+ "reward": 0.4182003140449524,
+ "reward_std": 0.3098836839199066,
+ "rewards/true_env_reward_fn/mean": 0.4182003140449524,
+ "rewards/true_env_reward_fn/std": 0.3098836839199066,
+ "step": 367,
+ "step_time": 4.085919256000125
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 94.0,
+ "completions/max_terminated_length": 94.0,
+ "completions/mean_length": 55.25,
+ "completions/mean_terminated_length": 55.25,
+ "completions/min_length": 18.0,
+ "completions/min_terminated_length": 18.0,
+ "entropy": 1.3936381936073303,
+ "epoch": 2.991869918699187,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15050898492336273,
+ "kl": 3.818520599452313e-05,
+ "learning_rate": 7.163093210489357e-11,
+ "loss": -0.18500229716300964,
+ "num_tokens": 1511243.0,
+ "reward": 0.30103665590286255,
+ "reward_std": 0.2349206805229187,
+ "rewards/true_env_reward_fn/mean": 0.30103665590286255,
+ "rewards/true_env_reward_fn/std": 0.2349206656217575,
+ "step": 368,
+ "step_time": 4.331018617000154
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 90.0,
+ "completions/max_terminated_length": 90.0,
+ "completions/mean_length": 67.875,
+ "completions/mean_terminated_length": 67.875,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.272651731967926,
+ "epoch": 3.0,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 0.0001919015048770234,
+ "kl": 2.4650291379657574e-05,
+ "learning_rate": 1.7908133902810874e-11,
+ "loss": 1.2321368103584973e-06,
+ "num_tokens": 1516382.0,
+ "reward": 0.4544333219528198,
+ "reward_std": 0.005238313227891922,
+ "rewards/true_env_reward_fn/mean": 0.4544333219528198,
+ "rewards/true_env_reward_fn/std": 0.005238313693553209,
+ "step": 369,
+ "step_time": 4.247185335001632
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 1516382,
+ "num_train_epochs": 3,
+ "save_steps": 50,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-369/training_args.bin b/checkpoint-369/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8d94c3c38f17faf8a60976b504514708acad4864
--- /dev/null
+++ b/checkpoint-369/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/checkpoint-50/README.md b/checkpoint-50/README.md
index 002e352e3f4655c9f00611b74eefa7b221963ec8..d43f5cce16542812c28205685858b78648c39bdb 100644
--- a/checkpoint-50/README.md
+++ b/checkpoint-50/README.md
@@ -1,14 +1,13 @@
---
-base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
+base_model: unsloth/Qwen2.5-1.5B-Instruct
library_name: peft
pipeline_tag: text-generation
tags:
-- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit
+- base_model:adapter:unsloth/Qwen2.5-1.5B-Instruct
- grpo
- lora
- transformers
- trl
-- unsloth
---
# Model Card for Model ID
@@ -207,4 +206,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
[More Information Needed]
### Framework versions
-- PEFT 0.18.1
\ No newline at end of file
+- PEFT 0.19.1
\ No newline at end of file
diff --git a/checkpoint-50/adapter_config.json b/checkpoint-50/adapter_config.json
index b79ee98cab9fe9bafd1f7a7f8f257f268bb548e8..1335f363e822e5c942fa1ace6e0eb5fabdd99524 100644
--- a/checkpoint-50/adapter_config.json
+++ b/checkpoint-50/adapter_config.json
@@ -2,12 +2,8 @@
"alora_invocation_tokens": null,
"alpha_pattern": {},
"arrow_config": null,
- "auto_mapping": {
- "base_model_class": "Qwen2ForCausalLM",
- "parent_library": "transformers.models.qwen2.modeling_qwen2",
- "unsloth_fixed": true
- },
- "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit",
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
"bias": "none",
"corda_config": null,
"ensure_weight_tying": false,
@@ -23,27 +19,24 @@
"lora_alpha": 16,
"lora_bias": false,
"lora_dropout": 0.05,
+ "lora_ga_config": null,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": null,
"peft_type": "LORA",
- "peft_version": "0.18.1",
+ "peft_version": "0.19.1",
"qalora_group_size": 16,
"r": 16,
"rank_pattern": {},
"revision": null,
"target_modules": [
- "up_proj",
"q_proj",
- "gate_proj",
- "down_proj",
- "v_proj",
- "o_proj",
- "k_proj"
+ "v_proj"
],
"target_parameters": null,
"task_type": "CAUSAL_LM",
"trainable_token_indices": null,
+ "use_bdlora": null,
"use_dora": false,
"use_qalora": false,
"use_rslora": false
diff --git a/checkpoint-50/adapter_model.safetensors b/checkpoint-50/adapter_model.safetensors
index 683d5cf7add087b14a090b68c576f014da4df889..41f4d0bcb31c19b0b40b39b0cea452fef7435087 100644
--- a/checkpoint-50/adapter_model.safetensors
+++ b/checkpoint-50/adapter_model.safetensors
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:67e2ea884d8b3bfcc42fd3329a428adfc3d07a2a7a760f48a52ca3189a6cfd73
-size 73911112
+oid sha256:c29d16da19e145b7fbc421d2926b57f8a64fb43aa41d6a7e0e650b362d91bdcc
+size 8731128
diff --git a/checkpoint-50/optimizer.pt b/checkpoint-50/optimizer.pt
index 4a7dcf99447d8d97062d34824f87aa05012634a2..624e64865fe18a4876906054cacb75458b072bd9 100644
--- a/checkpoint-50/optimizer.pt
+++ b/checkpoint-50/optimizer.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:7c5f63e2b2c962b91db7caa9e23538582983f974258bb97c877cdab2f4d8e9fc
-size 37969669
+oid sha256:f36ff735dcde2bd473a8af37048e6d1ef9de616653181fc0aedf95971b4e311b
+size 17526842
diff --git a/checkpoint-50/ref/adapter_config.json b/checkpoint-50/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/checkpoint-50/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-50/ref/adapter_model.safetensors b/checkpoint-50/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/checkpoint-50/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/checkpoint-50/rng_state.pth b/checkpoint-50/rng_state.pth
index 85db339bca5bb4b888dab41114dc171186184e83..1255bd47b09c1c1d5e66537d92eb0be204a3d8aa 100644
--- a/checkpoint-50/rng_state.pth
+++ b/checkpoint-50/rng_state.pth
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:05a231aa61f941db8d8f5362cb8515f1e52d83aa4b34475898d47482e916ec0a
-size 14645
+oid sha256:6a8029353fcc2917f4dbe146b9fe43c7d58980906ebdeb29c19d8a55b6a7e36b
+size 14244
diff --git a/checkpoint-50/scaler.pt b/checkpoint-50/scaler.pt
index 48ade11e4a25ce357344c0aca97f5e25df3bddc4..ac9feb4bad410a1c1ce4be45aa33258a89bedb39 100644
--- a/checkpoint-50/scaler.pt
+++ b/checkpoint-50/scaler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:5cd0e9d505fbc3f97feb166d29026132bdf14eb3e5c7ff77beebc303ee666f96
-size 1383
+oid sha256:bce7091777d309ad0996ba841e0cacd9295db515ae8e6e24f546718eefe0d105
+size 988
diff --git a/checkpoint-50/scheduler.pt b/checkpoint-50/scheduler.pt
index 1cc1d99eb96bbf00f4765eee5ddea7a03187689e..a0574e7184eee77da8cca88e473915c7f5203554 100644
--- a/checkpoint-50/scheduler.pt
+++ b/checkpoint-50/scheduler.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:01f6b7bc7d819d4a1796fc90fe9d691cb8930388438f3f5ea720c463f372ca5f
-size 1465
+oid sha256:12478c219de6b77a5dc1e7fcc9ef1b62a708c4dfc483dc14a2b66e2a1ade05dd
+size 1064
diff --git a/checkpoint-50/tokenizer.json b/checkpoint-50/tokenizer.json
index af979ec5282fda15d3de42a1719f7aff8f802448..34510ff0037cd50428af467a17ead5a96140a32c 100644
--- a/checkpoint-50/tokenizer.json
+++ b/checkpoint-50/tokenizer.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:1e2928ca692824ff18793d8aafde64d8320497b91445e558619e994c04e67c5d
-size 11422520
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/checkpoint-50/tokenizer_config.json b/checkpoint-50/tokenizer_config.json
index 15e9b47cf6034fb2a8724229947e2007d5758893..9aa5955b7dd3b15995a880b534c69ad538218d2d 100644
--- a/checkpoint-50/tokenizer_config.json
+++ b/checkpoint-50/tokenizer_config.json
@@ -5,197 +5,12 @@
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
- "is_local": false,
+ "is_local": true,
+ "local_files_only": false,
"model_max_length": 32768,
"pad_token": "<|im_end|>",
- "padding_side": "right",
+ "padding_side": "left",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
- "unk_token": null,
- "added_tokens_decoder": {
- "151643": {
- "content": "<|endoftext|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151644": {
- "content": "<|im_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151645": {
- "content": "<|im_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151646": {
- "content": "<|object_ref_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151647": {
- "content": "<|object_ref_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151648": {
- "content": "<|box_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151649": {
- "content": "<|box_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151650": {
- "content": "<|quad_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151651": {
- "content": "<|quad_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151652": {
- "content": "<|vision_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151653": {
- "content": "<|vision_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151654": {
- "content": "<|vision_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151655": {
- "content": "<|image_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151656": {
- "content": "<|video_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151657": {
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151658": {
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151659": {
- "content": "<|fim_prefix|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151660": {
- "content": "<|fim_middle|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151661": {
- "content": "<|fim_suffix|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151662": {
- "content": "<|fim_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151663": {
- "content": "<|repo_name|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151664": {
- "content": "<|file_sep|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151665": {
- "content": "<|PAD_TOKEN|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- }
- }
+ "unk_token": null
}
diff --git a/checkpoint-50/trainer_state.json b/checkpoint-50/trainer_state.json
index a9bd7ac86ecf56f482f318cea838ca2ec84638cc..a9aeab3accbd70e2a3a52a958bca65aca0cae400 100644
--- a/checkpoint-50/trainer_state.json
+++ b/checkpoint-50/trainer_state.json
@@ -2,7 +2,7 @@
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
- "epoch": 0.847457627118644,
+ "epoch": 0.4065040650406504,
"eval_steps": 500,
"global_step": 50,
"is_hyper_param_search": false,
@@ -15,26 +15,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 77.875,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 78.75,
- "completions/mean_terminated_length": 71.71428680419922,
- "completions/min_length": 50.0,
- "completions/min_terminated_length": 50.0,
- "epoch": 0.01694915254237288,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.005794263910502195,
- "kl": 0.0033121350570581853,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 78.0,
+ "completions/max_terminated_length": 78.0,
+ "completions/mean_length": 56.25,
+ "completions/mean_terminated_length": 56.25,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1208415031433105,
+ "epoch": 0.008130081300813009,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.2371738702058792,
+ "kl": 1.1247546808590414e-05,
"learning_rate": 0.0,
- "loss": 0.000168175611179322,
- "num_tokens": 5606.0,
- "reward": 0.1915000081062317,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.1915000081062317,
- "rewards/true_env_reward_fn/std": 0.2902457118034363,
- "step": 1
+ "loss": 0.019987963140010834,
+ "num_tokens": 2250.0,
+ "reward": 0.7777429223060608,
+ "reward_std": 0.14680756628513336,
+ "rewards/true_env_reward_fn/mean": 0.7777429223060608,
+ "rewards/true_env_reward_fn/std": 0.14680756628513336,
+ "step": 1,
+ "step_time": 3.622400252999796
},
{
"clip_ratio/high_max": 0.0,
@@ -42,26 +43,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 53.25,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 67.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 54.25,
- "completions/mean_terminated_length": 54.25,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "epoch": 0.03389830508474576,
+ "completions/max_length": 126.0,
+ "completions/max_terminated_length": 126.0,
+ "completions/mean_length": 72.875,
+ "completions/mean_terminated_length": 72.875,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 0.9768376648426056,
+ "epoch": 0.016260162601626018,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.3056783974170685,
- "kl": 0.012695180019363761,
- "learning_rate": 1.6666666666666665e-07,
- "loss": 0.005846855230629444,
- "num_tokens": 8560.0,
- "reward": 0.5156262516975403,
- "reward_std": 0.08491785824298859,
- "rewards/true_env_reward_fn/mean": 0.5156261920928955,
- "rewards/true_env_reward_fn/std": 0.3809922933578491,
- "step": 2
+ "grad_norm": 0.10255444794893265,
+ "kl": 9.209406016452704e-06,
+ "learning_rate": 2.1621621621621623e-08,
+ "loss": -0.16146813333034515,
+ "num_tokens": 8517.0,
+ "reward": 0.5471514463424683,
+ "reward_std": 0.19726651906967163,
+ "rewards/true_env_reward_fn/mean": 0.5471514463424683,
+ "rewards/true_env_reward_fn/std": 0.19726651906967163,
+ "step": 2,
+ "step_time": 5.979386726001394
},
{
"clip_ratio/high_max": 0.0,
@@ -69,26 +71,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 58.25,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 97.0,
- "completions/max_terminated_length": 97.0,
- "completions/mean_length": 59.25,
- "completions/mean_terminated_length": 59.25,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "epoch": 0.05084745762711865,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.42266136407852173,
- "kl": 0.005654739215970039,
- "learning_rate": 3.333333333333333e-07,
- "loss": -0.005579017102718353,
- "num_tokens": 14478.0,
- "reward": 0.11133332550525665,
- "reward_std": 0.2769676446914673,
- "rewards/true_env_reward_fn/mean": 0.11133334040641785,
- "rewards/true_env_reward_fn/std": 0.2668781280517578,
- "step": 3
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 67.125,
+ "completions/mean_terminated_length": 67.125,
+ "completions/min_length": 53.0,
+ "completions/min_terminated_length": 53.0,
+ "entropy": 1.324017882347107,
+ "epoch": 0.024390243902439025,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.13558730483055115,
+ "kl": 1.2776082257914823e-05,
+ "learning_rate": 4.3243243243243246e-08,
+ "loss": 0.008048340678215027,
+ "num_tokens": 11858.0,
+ "reward": 0.5399026870727539,
+ "reward_std": 0.04722921922802925,
+ "rewards/true_env_reward_fn/mean": 0.5399026870727539,
+ "rewards/true_env_reward_fn/std": 0.047229230403900146,
+ "step": 3,
+ "step_time": 3.6805073480009014
},
{
"clip_ratio/high_max": 0.0,
@@ -96,26 +99,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 56.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 115.0,
- "completions/max_terminated_length": 115.0,
- "completions/mean_length": 57.875,
- "completions/mean_terminated_length": 57.875,
- "completions/min_length": 23.0,
- "completions/min_terminated_length": 23.0,
- "epoch": 0.06779661016949153,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.010326593182981014,
- "kl": 0.009371287073008716,
- "learning_rate": 5e-07,
- "loss": 0.0005386476404964924,
- "num_tokens": 18437.0,
- "reward": 0.3857494592666626,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.3857494592666626,
- "rewards/true_env_reward_fn/std": 0.503252387046814,
- "step": 4
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 53.625,
+ "completions/mean_terminated_length": 53.625,
+ "completions/min_length": 31.0,
+ "completions/min_terminated_length": 31.0,
+ "entropy": 1.0729783773422241,
+ "epoch": 0.032520325203252036,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23012493550777435,
+ "kl": 1.0804946214193478e-05,
+ "learning_rate": 6.486486486486487e-08,
+ "loss": 0.13091428577899933,
+ "num_tokens": 15379.0,
+ "reward": 0.4351762533187866,
+ "reward_std": 0.2320314645767212,
+ "rewards/true_env_reward_fn/mean": 0.4351762533187866,
+ "rewards/true_env_reward_fn/std": 0.2320314645767212,
+ "step": 4,
+ "step_time": 3.421140036001816
},
{
"clip_ratio/high_max": 0.0,
@@ -123,26 +127,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 56.375,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 76.0,
- "completions/max_terminated_length": 76.0,
- "completions/mean_length": 57.375,
- "completions/mean_terminated_length": 57.375,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "epoch": 0.0847457627118644,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.006233581341803074,
- "kl": 0.004069901129696518,
- "learning_rate": 6.666666666666666e-07,
- "loss": 0.00020383310038596392,
- "num_tokens": 21800.0,
- "reward": 0.46670055389404297,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.46670055389404297,
- "rewards/true_env_reward_fn/std": 0.019635379314422607,
- "step": 5
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 57.25,
+ "completions/mean_terminated_length": 57.25,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.3000869154930115,
+ "epoch": 0.04065040650406504,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22867721319198608,
+ "kl": 1.2170262834843015e-05,
+ "learning_rate": 8.648648648648649e-08,
+ "loss": 0.08851668983697891,
+ "num_tokens": 19401.0,
+ "reward": 0.4031979441642761,
+ "reward_std": 0.32033228874206543,
+ "rewards/true_env_reward_fn/mean": 0.4031979441642761,
+ "rewards/true_env_reward_fn/std": 0.32033228874206543,
+ "step": 5,
+ "step_time": 3.977350764000221
},
{
"clip_ratio/high_max": 0.0,
@@ -150,26 +155,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 71.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 110.0,
- "completions/max_terminated_length": 110.0,
- "completions/mean_length": 72.0,
- "completions/mean_terminated_length": 72.0,
- "completions/min_length": 61.0,
- "completions/min_terminated_length": 61.0,
- "epoch": 0.1016949152542373,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 58.5,
+ "completions/mean_terminated_length": 58.5,
+ "completions/min_length": 27.0,
+ "completions/min_terminated_length": 27.0,
+ "entropy": 1.1719728112220764,
+ "epoch": 0.04878048780487805,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.24560949206352234,
- "kl": 0.00258735561510548,
- "learning_rate": 8.333333333333333e-07,
- "loss": 0.04410937801003456,
- "num_tokens": 26776.0,
- "reward": 0.32221275568008423,
- "reward_std": 0.07824588567018509,
- "rewards/true_env_reward_fn/mean": 0.32221275568008423,
- "rewards/true_env_reward_fn/std": 0.18206648528575897,
- "step": 6
+ "grad_norm": 0.16202858090400696,
+ "kl": 1.228428209287813e-05,
+ "learning_rate": 1.0810810810810811e-07,
+ "loss": 0.1666201949119568,
+ "num_tokens": 21253.0,
+ "reward": 0.5827490091323853,
+ "reward_std": 0.27126544713974,
+ "rewards/true_env_reward_fn/mean": 0.5827490091323853,
+ "rewards/true_env_reward_fn/std": 0.27126544713974,
+ "step": 6,
+ "step_time": 4.179320960000041
},
{
"clip_ratio/high_max": 0.0,
@@ -177,26 +183,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 43.375,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 53.0,
- "completions/max_terminated_length": 53.0,
- "completions/mean_length": 44.375,
- "completions/mean_terminated_length": 44.375,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "epoch": 0.11864406779661017,
+ "completions/max_length": 72.0,
+ "completions/max_terminated_length": 72.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.3112086653709412,
+ "epoch": 0.056910569105691054,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.5008023977279663,
- "kl": 0.008291650272440165,
- "learning_rate": 1e-06,
- "loss": -0.024788305163383484,
- "num_tokens": 30299.0,
- "reward": 0.5258656144142151,
- "reward_std": 0.10140049457550049,
- "rewards/true_env_reward_fn/mean": 0.5258656740188599,
- "rewards/true_env_reward_fn/std": 0.11995989084243774,
- "step": 7
+ "grad_norm": 0.20938768982887268,
+ "kl": 1.306734156969469e-05,
+ "learning_rate": 1.2972972972972974e-07,
+ "loss": 0.04748187214136124,
+ "num_tokens": 25726.0,
+ "reward": 0.2716812491416931,
+ "reward_std": 0.29254475235939026,
+ "rewards/true_env_reward_fn/mean": 0.2716812491416931,
+ "rewards/true_env_reward_fn/std": 0.29254478216171265,
+ "step": 7,
+ "step_time": 3.438178512999002
},
{
"clip_ratio/high_max": 0.0,
@@ -204,26 +211,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 39.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 40.5,
- "completions/mean_terminated_length": 40.5,
- "completions/min_length": 17.0,
- "completions/min_terminated_length": 17.0,
- "epoch": 0.13559322033898305,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.6174954175949097,
- "kl": 0.009604852180927992,
- "learning_rate": 9.991218658821608e-07,
- "loss": 0.04839690029621124,
- "num_tokens": 34051.0,
- "reward": 0.3403191864490509,
- "reward_std": 0.1679297685623169,
- "rewards/true_env_reward_fn/mean": 0.3403192162513733,
- "rewards/true_env_reward_fn/std": 0.28296828269958496,
- "step": 8
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 78.5,
+ "completions/mean_terminated_length": 78.5,
+ "completions/min_length": 65.0,
+ "completions/min_terminated_length": 65.0,
+ "entropy": 1.2046615481376648,
+ "epoch": 0.06504065040650407,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 7.857898890506476e-05,
+ "kl": 1.3628536180476658e-05,
+ "learning_rate": 1.5135135135135135e-07,
+ "loss": 6.865971045044716e-07,
+ "num_tokens": 30126.0,
+ "reward": 0.49959999322891235,
+ "reward_std": 0.02822280302643776,
+ "rewards/true_env_reward_fn/mean": 0.49959999322891235,
+ "rewards/true_env_reward_fn/std": 0.02822280302643776,
+ "step": 8,
+ "step_time": 4.324984626000514
},
{
"clip_ratio/high_max": 0.0,
@@ -231,26 +239,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 57.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 58.625,
- "completions/mean_terminated_length": 58.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "epoch": 0.15254237288135594,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.5500961542129517,
- "kl": 0.01022835448384285,
- "learning_rate": 9.964905480067584e-07,
- "loss": 0.004459276795387268,
- "num_tokens": 37992.0,
- "reward": 0.2750284969806671,
- "reward_std": 0.27525249123573303,
- "rewards/true_env_reward_fn/mean": 0.2750285267829895,
- "rewards/true_env_reward_fn/std": 0.433340847492218,
- "step": 9
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 62.75,
+ "completions/mean_terminated_length": 62.75,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2111859917640686,
+ "epoch": 0.07317073170731707,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11552055925130844,
+ "kl": 1.0166647598452982e-05,
+ "learning_rate": 1.7297297297297298e-07,
+ "loss": -0.03883127495646477,
+ "num_tokens": 33332.0,
+ "reward": 0.571246862411499,
+ "reward_std": 0.2893567681312561,
+ "rewards/true_env_reward_fn/mean": 0.571246862411499,
+ "rewards/true_env_reward_fn/std": 0.2893567681312561,
+ "step": 9,
+ "step_time": 3.651253555999574
},
{
"clip_ratio/high_max": 0.0,
@@ -258,26 +267,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 48.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 49.5,
- "completions/mean_terminated_length": 49.5,
- "completions/min_length": 34.0,
- "completions/min_terminated_length": 34.0,
- "epoch": 0.1694915254237288,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.005737450439482927,
- "kl": 0.005019246542360634,
- "learning_rate": 9.921152889737984e-07,
- "loss": 0.00024261200451292098,
- "num_tokens": 41444.0,
- "reward": 0.6353538036346436,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.6353538036346436,
- "rewards/true_env_reward_fn/std": 0.059336014091968536,
- "step": 10
+ "completions/max_length": 85.0,
+ "completions/max_terminated_length": 85.0,
+ "completions/mean_length": 58.125,
+ "completions/mean_terminated_length": 58.125,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.195803463459015,
+ "epoch": 0.08130081300813008,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14317302405834198,
+ "kl": 1.476421539337025e-05,
+ "learning_rate": 1.945945945945946e-07,
+ "loss": -0.034387920051813126,
+ "num_tokens": 36381.0,
+ "reward": 0.5593140125274658,
+ "reward_std": 0.42223072052001953,
+ "rewards/true_env_reward_fn/mean": 0.5593140125274658,
+ "rewards/true_env_reward_fn/std": 0.42223072052001953,
+ "step": 10,
+ "step_time": 3.8103441190014564
},
{
"clip_ratio/high_max": 0.0,
@@ -285,26 +295,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 55.75,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 56.75,
- "completions/mean_terminated_length": 56.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "epoch": 0.1864406779661017,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 61.625,
+ "completions/mean_terminated_length": 61.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.389159917831421,
+ "epoch": 0.08943089430894309,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.2932603061199188,
- "kl": 0.005889773019589484,
- "learning_rate": 9.860114570402054e-07,
- "loss": 0.03033295087516308,
- "num_tokens": 44386.0,
- "reward": 0.6600469350814819,
- "reward_std": 0.09155216068029404,
- "rewards/true_env_reward_fn/mean": 0.6600469350814819,
- "rewards/true_env_reward_fn/std": 0.12431291490793228,
- "step": 11
+ "grad_norm": 0.11816789954900742,
+ "kl": 1.0807365470100194e-05,
+ "learning_rate": 2.1621621621621622e-07,
+ "loss": 0.04055345058441162,
+ "num_tokens": 41830.0,
+ "reward": 0.12224999815225601,
+ "reward_std": 0.27913153171539307,
+ "rewards/true_env_reward_fn/mean": 0.12224999815225601,
+ "rewards/true_env_reward_fn/std": 0.27913153171539307,
+ "step": 11,
+ "step_time": 4.204996996000773
},
{
"clip_ratio/high_max": 0.0,
@@ -312,26 +323,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 68.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 93.0,
- "completions/max_terminated_length": 93.0,
- "completions/mean_length": 69.5,
- "completions/mean_terminated_length": 69.5,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "epoch": 0.2033898305084746,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.23511165380477905,
- "kl": 0.009554511751048267,
- "learning_rate": 9.78200492138261e-07,
- "loss": -0.02436758577823639,
- "num_tokens": 48618.0,
- "reward": 0.3695759177207947,
- "reward_std": 0.07969073951244354,
- "rewards/true_env_reward_fn/mean": 0.36957594752311707,
- "rewards/true_env_reward_fn/std": 0.1309438943862915,
- "step": 12
+ "completions/max_length": 106.0,
+ "completions/max_terminated_length": 106.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 54.0,
+ "completions/min_terminated_length": 54.0,
+ "entropy": 1.3866143822669983,
+ "epoch": 0.0975609756097561,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.19537723064422607,
+ "kl": 1.5072046608111123e-05,
+ "learning_rate": 2.3783783783783783e-07,
+ "loss": -0.07735465466976166,
+ "num_tokens": 47047.0,
+ "reward": 0.3571999967098236,
+ "reward_std": 0.18295250833034515,
+ "rewards/true_env_reward_fn/mean": 0.3571999967098236,
+ "rewards/true_env_reward_fn/std": 0.18295250833034515,
+ "step": 12,
+ "step_time": 4.775358541999594
},
{
"clip_ratio/high_max": 0.0,
@@ -339,26 +351,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 51.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 52.875,
- "completions/mean_terminated_length": 52.875,
- "completions/min_length": 27.0,
- "completions/min_terminated_length": 27.0,
- "epoch": 0.22033898305084745,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004275341983884573,
- "kl": 0.0036962208105251193,
- "learning_rate": 9.687098305670604e-07,
- "loss": 0.000169083708897233,
- "num_tokens": 52205.0,
- "reward": 0.4483333230018616,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4483333230018616,
- "rewards/true_env_reward_fn/std": 0.0,
- "step": 13
+ "completions/max_length": 86.0,
+ "completions/max_terminated_length": 86.0,
+ "completions/mean_length": 55.75,
+ "completions/mean_terminated_length": 55.75,
+ "completions/min_length": 38.0,
+ "completions/min_terminated_length": 38.0,
+ "entropy": 1.1633875966072083,
+ "epoch": 0.10569105691056911,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14852823317050934,
+ "kl": 1.4038786503078882e-05,
+ "learning_rate": 2.594594594594595e-07,
+ "loss": -0.04705440253019333,
+ "num_tokens": 51521.0,
+ "reward": 0.44465911388397217,
+ "reward_std": 0.15160730481147766,
+ "rewards/true_env_reward_fn/mean": 0.44465911388397217,
+ "rewards/true_env_reward_fn/std": 0.15160730481147766,
+ "step": 13,
+ "step_time": 4.072596639998665
},
{
"clip_ratio/high_max": 0.0,
@@ -366,26 +379,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 62.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 72.0,
- "completions/max_terminated_length": 72.0,
- "completions/mean_length": 63.625,
- "completions/mean_terminated_length": 63.625,
- "completions/min_length": 54.0,
- "completions/min_terminated_length": 54.0,
- "epoch": 0.23728813559322035,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.27066466212272644,
- "kl": 0.003385908610653132,
- "learning_rate": 9.575728086215091e-07,
- "loss": -0.012835565023124218,
- "num_tokens": 57102.0,
- "reward": 0.3044250011444092,
- "reward_std": 0.06293118000030518,
- "rewards/true_env_reward_fn/mean": 0.3044250011444092,
- "rewards/true_env_reward_fn/std": 0.30575329065322876,
- "step": 14
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 60.375,
+ "completions/mean_terminated_length": 60.375,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.2392634153366089,
+ "epoch": 0.11382113821138211,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23616985976696014,
+ "kl": 1.3279905488161603e-05,
+ "learning_rate": 2.810810810810811e-07,
+ "loss": -0.021731968969106674,
+ "num_tokens": 55556.0,
+ "reward": 0.4130214750766754,
+ "reward_std": 0.43705809116363525,
+ "rewards/true_env_reward_fn/mean": 0.4130214750766754,
+ "rewards/true_env_reward_fn/std": 0.43705806136131287,
+ "step": 14,
+ "step_time": 3.7906999759998143
},
{
"clip_ratio/high_max": 0.0,
@@ -393,26 +407,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 64.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 111.0,
- "completions/max_terminated_length": 111.0,
- "completions/mean_length": 65.0,
- "completions/mean_terminated_length": 65.0,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "epoch": 0.2542372881355932,
+ "completions/max_length": 75.0,
+ "completions/max_terminated_length": 75.0,
+ "completions/mean_length": 61.5,
+ "completions/mean_terminated_length": 61.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.1012902855873108,
+ "epoch": 0.12195121951219512,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.27638667821884155,
- "kl": 0.004561240901239216,
- "learning_rate": 9.448285454973737e-07,
- "loss": -0.006410777568817139,
- "num_tokens": 62894.0,
- "reward": 0.25437501072883606,
- "reward_std": 0.12575000524520874,
- "rewards/true_env_reward_fn/mean": 0.25437501072883606,
- "rewards/true_env_reward_fn/std": 0.2772188186645508,
- "step": 15
+ "grad_norm": 0.11504171043634415,
+ "kl": 1.1161580914631486e-05,
+ "learning_rate": 3.027027027027027e-07,
+ "loss": -0.03352496027946472,
+ "num_tokens": 58644.0,
+ "reward": 0.54444819688797,
+ "reward_std": 0.2691938281059265,
+ "rewards/true_env_reward_fn/mean": 0.54444819688797,
+ "rewards/true_env_reward_fn/std": 0.2691938281059265,
+ "step": 15,
+ "step_time": 3.427628186998845
},
{
"clip_ratio/high_max": 0.0,
@@ -420,26 +435,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 39.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 54.0,
- "completions/max_terminated_length": 54.0,
- "completions/mean_length": 40.5,
- "completions/mean_terminated_length": 40.5,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "epoch": 0.2711864406779661,
+ "completions/max_length": 110.0,
+ "completions/max_terminated_length": 110.0,
+ "completions/mean_length": 66.125,
+ "completions/mean_terminated_length": 66.125,
+ "completions/min_length": 32.0,
+ "completions/min_terminated_length": 32.0,
+ "entropy": 1.1984660625457764,
+ "epoch": 0.13008130081300814,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.3701099753379822,
- "kl": 0.02095008222386241,
- "learning_rate": 9.305218058836776e-07,
- "loss": 0.021562710404396057,
- "num_tokens": 64226.0,
- "reward": 0.7455551624298096,
- "reward_std": 0.08850937336683273,
- "rewards/true_env_reward_fn/mean": 0.7455551624298096,
- "rewards/true_env_reward_fn/std": 0.16582044959068298,
- "step": 16
+ "grad_norm": 0.12652896344661713,
+ "kl": 1.292689512411016e-05,
+ "learning_rate": 3.243243243243243e-07,
+ "loss": -0.08787620067596436,
+ "num_tokens": 62361.0,
+ "reward": 0.46189582347869873,
+ "reward_std": 0.23188425600528717,
+ "rewards/true_env_reward_fn/mean": 0.46189582347869873,
+ "rewards/true_env_reward_fn/std": 0.23188428580760956,
+ "step": 16,
+ "step_time": 4.776189491001787
},
{
"clip_ratio/high_max": 0.0,
@@ -447,26 +463,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 52.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 53.875,
- "completions/mean_terminated_length": 53.875,
- "completions/min_length": 22.0,
- "completions/min_terminated_length": 22.0,
- "epoch": 0.288135593220339,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 58.25,
+ "completions/mean_terminated_length": 58.25,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.1009634137153625,
+ "epoch": 0.13821138211382114,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.45302459597587585,
- "kl": 0.0036311341682448983,
- "learning_rate": 9.147028427251009e-07,
- "loss": -0.022464489564299583,
- "num_tokens": 68281.0,
- "reward": 0.2526249885559082,
- "reward_std": 0.10478603839874268,
- "rewards/true_env_reward_fn/mean": 0.2526249885559082,
- "rewards/true_env_reward_fn/std": 0.25019291043281555,
- "step": 17
+ "grad_norm": 0.1329507976770401,
+ "kl": 1.1219160569453379e-05,
+ "learning_rate": 3.4594594594594597e-07,
+ "loss": 0.07219867408275604,
+ "num_tokens": 65899.0,
+ "reward": 0.65608811378479,
+ "reward_std": 0.2155800759792328,
+ "rewards/true_env_reward_fn/mean": 0.65608811378479,
+ "rewards/true_env_reward_fn/std": 0.21558009088039398,
+ "step": 17,
+ "step_time": 3.525365152998347
},
{
"clip_ratio/high_max": 0.0,
@@ -474,53 +491,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 44.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 56.0,
- "completions/max_terminated_length": 56.0,
- "completions/mean_length": 45.625,
- "completions/mean_terminated_length": 45.625,
+ "completions/max_length": 89.0,
+ "completions/max_terminated_length": 89.0,
+ "completions/mean_length": 51.5,
+ "completions/mean_terminated_length": 51.5,
"completions/min_length": 29.0,
"completions/min_terminated_length": 29.0,
- "epoch": 0.3050847457627119,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.006607957649976015,
- "kl": 0.006444428232498467,
- "learning_rate": 8.974272207066767e-07,
- "loss": 0.00030619491008110344,
- "num_tokens": 71790.0,
- "reward": 0.5672204494476318,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.5672204494476318,
- "rewards/true_env_reward_fn/std": 0.13217361271381378,
- "step": 18
- },
- {
- "clip_ratio/high_max": 0.0,
- "clip_ratio/high_mean": 0.0,
- "clip_ratio/low_mean": 0.0,
- "clip_ratio/low_min": 0.0,
- "clip_ratio/region_mean": 0.0,
- "completion_length": 69.875,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 67.0,
- "completions/mean_length": 70.75,
- "completions/mean_terminated_length": 62.57143020629883,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "epoch": 0.3220338983050847,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004781526047736406,
- "kl": 0.004017786588519812,
- "learning_rate": 8.7875562108081e-07,
- "loss": 0.00019913529104087502,
- "num_tokens": 76060.0,
- "reward": 0.4792916774749756,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.4792916774749756,
- "rewards/true_env_reward_fn/std": 0.03817380219697952,
- "step": 19
+ "entropy": 1.1671696901321411,
+ "epoch": 0.14634146341463414,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15754961967468262,
+ "kl": 1.3107276572554838e-05,
+ "learning_rate": 3.6756756756756757e-07,
+ "loss": -0.016363894566893578,
+ "num_tokens": 70155.0,
+ "reward": 0.3013000190258026,
+ "reward_std": 0.2883487343788147,
+ "rewards/true_env_reward_fn/mean": 0.3013000190258026,
+ "rewards/true_env_reward_fn/std": 0.2883487641811371,
+ "step": 18,
+ "step_time": 4.143123763000403
},
{
"clip_ratio/high_max": 0.0,
@@ -528,26 +519,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 61.125,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 84.0,
- "completions/max_terminated_length": 84.0,
- "completions/mean_length": 62.125,
- "completions/mean_terminated_length": 62.125,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "epoch": 0.3389830508474576,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004737756680697203,
- "kl": 0.004095949116162956,
- "learning_rate": 8.587536285221655e-07,
- "loss": 0.00020718023006338626,
- "num_tokens": 80025.0,
- "reward": 0.5769287347793579,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.5769287347793579,
- "rewards/true_env_reward_fn/std": 0.12179497629404068,
- "step": 20
+ "completions/max_length": 111.0,
+ "completions/max_terminated_length": 111.0,
+ "completions/mean_length": 70.25,
+ "completions/mean_terminated_length": 70.25,
+ "completions/min_length": 48.0,
+ "completions/min_terminated_length": 48.0,
+ "entropy": 1.266749083995819,
+ "epoch": 0.15447154471544716,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.11665906757116318,
+ "kl": 1.2845626315538539e-05,
+ "learning_rate": 3.891891891891892e-07,
+ "loss": -0.11013027280569077,
+ "num_tokens": 73389.0,
+ "reward": 0.6058553457260132,
+ "reward_std": 0.11022671312093735,
+ "rewards/true_env_reward_fn/mean": 0.6058553457260132,
+ "rewards/true_env_reward_fn/std": 0.11022673547267914,
+ "step": 19,
+ "step_time": 4.701202698999623
},
{
"clip_ratio/high_max": 0.0,
@@ -555,26 +547,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 37.25,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 55.0,
- "completions/max_terminated_length": 55.0,
- "completions/mean_length": 38.25,
- "completions/mean_terminated_length": 38.25,
- "completions/min_length": 24.0,
- "completions/min_terminated_length": 24.0,
- "epoch": 0.3559322033898305,
- "frac_reward_zero_std": 0.0,
- "grad_norm": 0.6817008852958679,
- "kl": 0.03418115433305502,
- "learning_rate": 8.374915007591052e-07,
- "loss": -0.1671726405620575,
- "num_tokens": 81335.0,
- "reward": 0.6582077145576477,
- "reward_std": 0.20000000298023224,
- "rewards/true_env_reward_fn/mean": 0.6582076549530029,
- "rewards/true_env_reward_fn/std": 0.21290148794651031,
- "step": 21
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 40.0,
+ "completions/min_terminated_length": 40.0,
+ "entropy": 1.1111925840377808,
+ "epoch": 0.16260162601626016,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.1429353505373001,
+ "kl": 8.694359621586045e-06,
+ "learning_rate": 4.108108108108108e-07,
+ "loss": -0.05066477507352829,
+ "num_tokens": 77594.0,
+ "reward": 0.4271581172943115,
+ "reward_std": 0.050101421773433685,
+ "rewards/true_env_reward_fn/mean": 0.4271581172943115,
+ "rewards/true_env_reward_fn/std": 0.05010143294930458,
+ "step": 20,
+ "step_time": 3.220270914998764
},
{
"clip_ratio/high_max": 0.0,
@@ -582,26 +575,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 42.375,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 43.375,
- "completions/mean_terminated_length": 43.375,
- "completions/min_length": 21.0,
- "completions/min_terminated_length": 21.0,
- "epoch": 0.3728813559322034,
+ "completions/max_length": 68.0,
+ "completions/max_terminated_length": 68.0,
+ "completions/mean_length": 46.0,
+ "completions/mean_terminated_length": 46.0,
+ "completions/min_length": 19.0,
+ "completions/min_terminated_length": 19.0,
+ "entropy": 1.4938308596611023,
+ "epoch": 0.17073170731707318,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.3072507679462433,
- "kl": 0.009053651010617614,
- "learning_rate": 8.150439217908556e-07,
- "loss": 0.03465559333562851,
- "num_tokens": 83574.0,
- "reward": 0.5414723753929138,
- "reward_std": 0.08850937336683273,
- "rewards/true_env_reward_fn/mean": 0.5414724349975586,
- "rewards/true_env_reward_fn/std": 0.1527864933013916,
- "step": 22
+ "grad_norm": 0.16324248909950256,
+ "kl": 1.1220067335671047e-05,
+ "learning_rate": 4.3243243243243244e-07,
+ "loss": 0.023503631353378296,
+ "num_tokens": 83298.0,
+ "reward": 0.1186770498752594,
+ "reward_std": 0.16449356079101562,
+ "rewards/true_env_reward_fn/mean": 0.1186770498752594,
+ "rewards/true_env_reward_fn/std": 0.16449356079101562,
+ "step": 21,
+ "step_time": 3.451675898999383
},
{
"clip_ratio/high_max": 0.0,
@@ -609,26 +603,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 56.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 87.0,
- "completions/max_terminated_length": 87.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "epoch": 0.3898305084745763,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.23083828389644623,
- "kl": 0.005918518756516278,
- "learning_rate": 7.91489739557236e-07,
- "loss": 0.0658169612288475,
- "num_tokens": 86863.0,
- "reward": 0.6125180721282959,
- "reward_std": 0.004953920841217041,
- "rewards/true_env_reward_fn/mean": 0.6125180721282959,
- "rewards/true_env_reward_fn/std": 0.18071529269218445,
- "step": 23
+ "completions/max_length": 122.0,
+ "completions/max_terminated_length": 122.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.2311039566993713,
+ "epoch": 0.17886178861788618,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.23201963305473328,
+ "kl": 1.2657743809540989e-05,
+ "learning_rate": 4.54054054054054e-07,
+ "loss": 0.20273712277412415,
+ "num_tokens": 87825.0,
+ "reward": 0.3444172441959381,
+ "reward_std": 0.4508652687072754,
+ "rewards/true_env_reward_fn/mean": 0.3444172441959381,
+ "rewards/true_env_reward_fn/std": 0.450865238904953,
+ "step": 22,
+ "step_time": 5.440214132999245
},
{
"clip_ratio/high_max": 0.0,
@@ -636,26 +631,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 45.25,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 65.0,
- "completions/max_terminated_length": 65.0,
- "completions/mean_length": 46.25,
- "completions/mean_terminated_length": 46.25,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "epoch": 0.4067796610169492,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 66.875,
+ "completions/mean_terminated_length": 66.875,
+ "completions/min_length": 41.0,
+ "completions/min_terminated_length": 41.0,
+ "entropy": 1.3744811415672302,
+ "epoch": 0.18699186991869918,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.32350578904151917,
- "kl": 0.0071139674400910735,
- "learning_rate": 7.669116889823954e-07,
- "loss": -0.03691143915057182,
- "num_tokens": 90973.0,
- "reward": 0.3783833384513855,
- "reward_std": 0.15121765434741974,
- "rewards/true_env_reward_fn/mean": 0.3783833384513855,
- "rewards/true_env_reward_fn/std": 0.29255327582359314,
- "step": 24
+ "grad_norm": 0.12073361128568649,
+ "kl": 1.356211032543797e-05,
+ "learning_rate": 4.7567567567567566e-07,
+ "loss": -0.06243758648633957,
+ "num_tokens": 92940.0,
+ "reward": 0.28657954931259155,
+ "reward_std": 0.19488918781280518,
+ "rewards/true_env_reward_fn/mean": 0.28657954931259155,
+ "rewards/true_env_reward_fn/std": 0.19488917291164398,
+ "step": 23,
+ "step_time": 4.085832714999924
},
{
"clip_ratio/high_max": 0.0,
@@ -663,26 +659,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 66.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "epoch": 0.423728813559322,
+ "completions/max_length": 100.0,
+ "completions/max_terminated_length": 100.0,
+ "completions/mean_length": 68.875,
+ "completions/mean_terminated_length": 68.875,
+ "completions/min_length": 23.0,
+ "completions/min_terminated_length": 23.0,
+ "entropy": 1.3229535818099976,
+ "epoch": 0.1951219512195122,
"frac_reward_zero_std": 1.0,
- "grad_norm": 0.003748361486941576,
- "kl": 0.0027431638445705175,
- "learning_rate": 7.413961013653725e-07,
- "loss": 0.0001304084580624476,
- "num_tokens": 96061.0,
- "reward": 0.46299999952316284,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.46299999952316284,
- "rewards/true_env_reward_fn/std": 0.0,
- "step": 25
+ "grad_norm": 8.64079047460109e-05,
+ "kl": 1.1639681815722724e-05,
+ "learning_rate": 4.972972972972973e-07,
+ "loss": 5.819025545861223e-07,
+ "num_tokens": 94923.0,
+ "reward": 0.7253252267837524,
+ "reward_std": 0.046159788966178894,
+ "rewards/true_env_reward_fn/mean": 0.7253252267837524,
+ "rewards/true_env_reward_fn/std": 0.046159788966178894,
+ "step": 24,
+ "step_time": 4.218084741001803
},
{
"clip_ratio/high_max": 0.0,
@@ -690,26 +687,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 66.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 78.0,
- "completions/max_terminated_length": 78.0,
- "completions/mean_length": 67.5,
- "completions/mean_terminated_length": 67.5,
- "completions/min_length": 57.0,
- "completions/min_terminated_length": 57.0,
- "epoch": 0.4406779661016949,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004255079198628664,
- "kl": 0.003476584271993488,
- "learning_rate": 7.150326011382603e-07,
- "loss": 0.00017167326586786658,
- "num_tokens": 99553.0,
- "reward": 0.569595456123352,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.569595456123352,
- "rewards/true_env_reward_fn/std": 0.12963464856147766,
- "step": 26
+ "completions/max_length": 113.0,
+ "completions/max_terminated_length": 113.0,
+ "completions/mean_length": 76.375,
+ "completions/mean_terminated_length": 76.375,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.3325599431991577,
+ "epoch": 0.2032520325203252,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.17998270690441132,
+ "kl": 1.4024041774973739e-05,
+ "learning_rate": 5.18918918918919e-07,
+ "loss": 0.13167564570903778,
+ "num_tokens": 99178.0,
+ "reward": 0.44252532720565796,
+ "reward_std": 0.1883804053068161,
+ "rewards/true_env_reward_fn/mean": 0.44252532720565796,
+ "rewards/true_env_reward_fn/std": 0.1883804053068161,
+ "step": 25,
+ "step_time": 4.84537445100068
},
{
"clip_ratio/high_max": 0.0,
@@ -717,26 +715,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 61.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 101.0,
- "completions/max_terminated_length": 101.0,
- "completions/mean_length": 62.875,
- "completions/mean_terminated_length": 62.875,
- "completions/min_length": 36.0,
- "completions/min_terminated_length": 36.0,
- "epoch": 0.4576271186440678,
+ "completions/max_length": 88.0,
+ "completions/max_terminated_length": 88.0,
+ "completions/mean_length": 64.375,
+ "completions/mean_terminated_length": 64.375,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.2788519263267517,
+ "epoch": 0.21138211382113822,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.3469686210155487,
- "kl": 0.011416598339565098,
- "learning_rate": 6.87913791057119e-07,
- "loss": -0.02214471995830536,
- "num_tokens": 103324.0,
- "reward": 0.47296762466430664,
- "reward_std": 0.08224812895059586,
- "rewards/true_env_reward_fn/mean": 0.47296762466430664,
- "rewards/true_env_reward_fn/std": 0.11086145788431168,
- "step": 27
+ "grad_norm": 0.1275048851966858,
+ "kl": 1.1262640327913687e-05,
+ "learning_rate": 5.405405405405405e-07,
+ "loss": -0.010535649955272675,
+ "num_tokens": 102353.0,
+ "reward": 0.3852383494377136,
+ "reward_std": 0.2447713315486908,
+ "rewards/true_env_reward_fn/mean": 0.3852383494377136,
+ "rewards/true_env_reward_fn/std": 0.244771346449852,
+ "step": 26,
+ "step_time": 3.80895136899926
},
{
"clip_ratio/high_max": 0.0,
@@ -744,26 +743,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 60.5,
"completions/clipped_ratio": 0.0,
"completions/max_length": 73.0,
"completions/max_terminated_length": 73.0,
- "completions/mean_length": 61.5,
- "completions/mean_terminated_length": 61.5,
- "completions/min_length": 45.0,
- "completions/min_terminated_length": 45.0,
- "epoch": 0.4745762711864407,
+ "completions/mean_length": 63.25,
+ "completions/mean_terminated_length": 63.25,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.0208025872707367,
+ "epoch": 0.21951219512195122,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.30816155672073364,
- "kl": 0.005478900973685086,
- "learning_rate": 6.601349269314187e-07,
- "loss": -0.006151705980300903,
- "num_tokens": 107656.0,
- "reward": 0.30828386545181274,
- "reward_std": 0.12210957705974579,
- "rewards/true_env_reward_fn/mean": 0.30828386545181274,
- "rewards/true_env_reward_fn/std": 0.24754488468170166,
- "step": 28
+ "grad_norm": 0.14226751029491425,
+ "kl": 1.4639559594797902e-05,
+ "learning_rate": 5.621621621621622e-07,
+ "loss": -0.05629514902830124,
+ "num_tokens": 103867.0,
+ "reward": 0.8898874521255493,
+ "reward_std": 0.1414213478565216,
+ "rewards/true_env_reward_fn/mean": 0.8898874521255493,
+ "rewards/true_env_reward_fn/std": 0.1414213478565216,
+ "step": 27,
+ "step_time": 3.227140603999942
},
{
"clip_ratio/high_max": 0.0,
@@ -771,26 +771,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 66.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 67.0,
- "completions/mean_terminated_length": 67.0,
- "completions/min_length": 39.0,
- "completions/min_terminated_length": 39.0,
- "epoch": 0.4915254237288136,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.3804105520248413,
- "kl": 0.006112040835432708,
- "learning_rate": 6.317935830345338e-07,
- "loss": 0.06503728777170181,
- "num_tokens": 110276.0,
- "reward": 0.4685196876525879,
- "reward_std": 0.16734644770622253,
- "rewards/true_env_reward_fn/mean": 0.4685196876525879,
- "rewards/true_env_reward_fn/std": 0.2246716320514679,
- "step": 29
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 66.5,
+ "completions/mean_terminated_length": 66.5,
+ "completions/min_length": 55.0,
+ "completions/min_terminated_length": 55.0,
+ "entropy": 1.278637707233429,
+ "epoch": 0.22764227642276422,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.6319210822694e-05,
+ "kl": 1.394796117892838e-05,
+ "learning_rate": 5.837837837837838e-07,
+ "loss": 6.984611218285863e-07,
+ "num_tokens": 108511.0,
+ "reward": 0.5384680032730103,
+ "reward_std": 0.06977442651987076,
+ "rewards/true_env_reward_fn/mean": 0.5384680032730103,
+ "rewards/true_env_reward_fn/std": 0.06977442651987076,
+ "step": 28,
+ "step_time": 3.3963304120006796
},
{
"clip_ratio/high_max": 0.0,
@@ -798,26 +799,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 50.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 51.5,
- "completions/mean_terminated_length": 51.5,
- "completions/min_length": 33.0,
- "completions/min_terminated_length": 33.0,
- "epoch": 0.5084745762711864,
+ "completions/max_length": 97.0,
+ "completions/max_terminated_length": 97.0,
+ "completions/mean_length": 73.5,
+ "completions/mean_terminated_length": 73.5,
+ "completions/min_length": 47.0,
+ "completions/min_terminated_length": 47.0,
+ "entropy": 1.2547507286071777,
+ "epoch": 0.23577235772357724,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.28046900033950806,
- "kl": 0.006454460555687547,
- "learning_rate": 6.029893093705491e-07,
- "loss": 0.028410714119672775,
- "num_tokens": 112852.0,
- "reward": 0.5688927173614502,
- "reward_std": 0.0031989715062081814,
- "rewards/true_env_reward_fn/mean": 0.5688927173614502,
- "rewards/true_env_reward_fn/std": 0.08971051126718521,
- "step": 30
+ "grad_norm": 0.10846269875764847,
+ "kl": 1.5149432329053525e-05,
+ "learning_rate": 6.054054054054054e-07,
+ "loss": 0.004249647259712219,
+ "num_tokens": 111323.0,
+ "reward": 0.6256026029586792,
+ "reward_std": 0.350762277841568,
+ "rewards/true_env_reward_fn/mean": 0.6256026029586792,
+ "rewards/true_env_reward_fn/std": 0.350762277841568,
+ "step": 29,
+ "step_time": 4.2103285969988065
},
{
"clip_ratio/high_max": 0.0,
@@ -825,26 +827,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 65.75,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 95.0,
- "completions/max_terminated_length": 95.0,
- "completions/mean_length": 66.75,
- "completions/mean_terminated_length": 66.75,
- "completions/min_length": 42.0,
- "completions/min_terminated_length": 42.0,
- "epoch": 0.5254237288135594,
+ "completions/max_length": 77.0,
+ "completions/max_terminated_length": 77.0,
+ "completions/mean_length": 56.625,
+ "completions/mean_terminated_length": 56.625,
+ "completions/min_length": 34.0,
+ "completions/min_terminated_length": 34.0,
+ "entropy": 1.4687196612358093,
+ "epoch": 0.24390243902439024,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.2755625247955322,
- "kl": 0.01492428407073021,
- "learning_rate": 5.738232820012407e-07,
- "loss": 0.02498520165681839,
- "num_tokens": 116662.0,
- "reward": 0.5067077875137329,
- "reward_std": 0.14062203466892242,
- "rewards/true_env_reward_fn/mean": 0.5067077875137329,
- "rewards/true_env_reward_fn/std": 0.4168119430541992,
- "step": 31
+ "grad_norm": 0.13451272249221802,
+ "kl": 1.3284446140460204e-05,
+ "learning_rate": 6.27027027027027e-07,
+ "loss": 0.05542291700839996,
+ "num_tokens": 115976.0,
+ "reward": 0.3901680111885071,
+ "reward_std": 0.2995865046977997,
+ "rewards/true_env_reward_fn/mean": 0.3901680111885071,
+ "rewards/true_env_reward_fn/std": 0.2995865046977997,
+ "step": 30,
+ "step_time": 3.5506420210003853
},
{
"clip_ratio/high_max": 0.0,
@@ -852,26 +855,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 58.125,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 116.0,
- "completions/max_terminated_length": 116.0,
- "completions/mean_length": 59.125,
- "completions/mean_terminated_length": 59.125,
- "completions/min_length": 22.0,
- "completions/min_terminated_length": 22.0,
- "epoch": 0.5423728813559322,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.32277289032936096,
- "kl": 0.009102466516196728,
- "learning_rate": 5.443979476614674e-07,
- "loss": -0.07510065287351608,
- "num_tokens": 119887.0,
- "reward": 0.5017499327659607,
- "reward_std": 0.10000000149011612,
- "rewards/true_env_reward_fn/mean": 0.5017499327659607,
- "rewards/true_env_reward_fn/std": 0.14494670927524567,
- "step": 32
+ "completions/max_length": 98.0,
+ "completions/max_terminated_length": 98.0,
+ "completions/mean_length": 76.875,
+ "completions/mean_terminated_length": 76.875,
+ "completions/min_length": 52.0,
+ "completions/min_terminated_length": 52.0,
+ "entropy": 1.2640270590782166,
+ "epoch": 0.25203252032520324,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.18553969264030457,
+ "kl": 1.2505860468081664e-05,
+ "learning_rate": 6.486486486486486e-07,
+ "loss": -0.015417251735925674,
+ "num_tokens": 118471.0,
+ "reward": 0.6587758660316467,
+ "reward_std": 0.14417217671871185,
+ "rewards/true_env_reward_fn/mean": 0.6587758660316467,
+ "rewards/true_env_reward_fn/std": 0.14417219161987305,
+ "step": 31,
+ "step_time": 4.198089399002129
},
{
"clip_ratio/high_max": 0.0,
@@ -879,26 +883,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 48.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 49.0,
- "completions/mean_terminated_length": 49.0,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "epoch": 0.559322033898305,
+ "completions/max_length": 84.0,
+ "completions/max_terminated_length": 84.0,
+ "completions/mean_length": 56.0,
+ "completions/mean_terminated_length": 56.0,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 1.5262224078178406,
+ "epoch": 0.2601626016260163,
"frac_reward_zero_std": 0.0,
- "grad_norm": 0.5441656708717346,
- "kl": 0.018496012780815363,
- "learning_rate": 5.148166639112799e-07,
- "loss": -0.22181616723537445,
- "num_tokens": 123475.0,
- "reward": 0.21720826625823975,
- "reward_std": 0.1699700504541397,
- "rewards/true_env_reward_fn/mean": 0.21720826625823975,
- "rewards/true_env_reward_fn/std": 0.30866101384162903,
- "step": 33
+ "grad_norm": 0.2357814759016037,
+ "kl": 1.6242850506387185e-05,
+ "learning_rate": 6.702702702702702e-07,
+ "loss": 0.031210914254188538,
+ "num_tokens": 123923.0,
+ "reward": 0.0943702906370163,
+ "reward_std": 0.1497660130262375,
+ "rewards/true_env_reward_fn/mean": 0.0943702906370163,
+ "rewards/true_env_reward_fn/std": 0.1497660130262375,
+ "step": 32,
+ "step_time": 3.978757984001277
},
{
"clip_ratio/high_max": 0.0,
@@ -906,26 +911,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 54.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 68.0,
- "completions/max_terminated_length": 68.0,
- "completions/mean_length": 55.875,
- "completions/mean_terminated_length": 55.875,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "epoch": 0.576271186440678,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.2937100827693939,
- "kl": 0.005296725546941161,
- "learning_rate": 4.851833360887201e-07,
- "loss": 0.023734737187623978,
- "num_tokens": 128010.0,
- "reward": 0.4385499954223633,
- "reward_std": 0.12575000524520874,
- "rewards/true_env_reward_fn/mean": 0.43855002522468567,
- "rewards/true_env_reward_fn/std": 0.22344137728214264,
- "step": 34
+ "completions/max_length": 82.0,
+ "completions/max_terminated_length": 82.0,
+ "completions/mean_length": 59.5,
+ "completions/mean_terminated_length": 59.5,
+ "completions/min_length": 20.0,
+ "completions/min_terminated_length": 20.0,
+ "entropy": 0.9924907088279724,
+ "epoch": 0.2682926829268293,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20393438637256622,
+ "kl": 1.181096149593941e-05,
+ "learning_rate": 6.918918918918919e-07,
+ "loss": -0.0068489015102386475,
+ "num_tokens": 130831.0,
+ "reward": 0.20862048864364624,
+ "reward_std": 0.2418184131383896,
+ "rewards/true_env_reward_fn/mean": 0.20862048864364624,
+ "rewards/true_env_reward_fn/std": 0.2418184131383896,
+ "step": 33,
+ "step_time": 4.237411461999727
},
{
"clip_ratio/high_max": 0.0,
@@ -933,26 +939,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 66.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 91.0,
- "completions/max_terminated_length": 91.0,
- "completions/mean_length": 67.625,
- "completions/mean_terminated_length": 67.625,
- "completions/min_length": 51.0,
- "completions/min_terminated_length": 51.0,
- "epoch": 0.5932203389830508,
+ "completions/max_length": 184.0,
+ "completions/max_terminated_length": 184.0,
+ "completions/mean_length": 105.0,
+ "completions/mean_terminated_length": 105.0,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.4909422397613525,
+ "epoch": 0.2764227642276423,
"frac_reward_zero_std": 1.0,
- "grad_norm": 0.00509282574057579,
- "kl": 0.004073156334925443,
- "learning_rate": 4.556020523385326e-07,
- "loss": 0.000214075407711789,
- "num_tokens": 131843.0,
- "reward": 0.6353538036346436,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.6353538036346436,
- "rewards/true_env_reward_fn/std": 0.059336014091968536,
- "step": 35
+ "grad_norm": 7.714608364040032e-05,
+ "kl": 1.3747331649938133e-05,
+ "learning_rate": 7.135135135135134e-07,
+ "loss": 6.856024583612452e-07,
+ "num_tokens": 138663.0,
+ "reward": 0.1821666657924652,
+ "reward_std": 0.2963036298751831,
+ "rewards/true_env_reward_fn/mean": 0.1821666657924652,
+ "rewards/true_env_reward_fn/std": 0.2963036298751831,
+ "step": 34,
+ "step_time": 8.45711429900075
},
{
"clip_ratio/high_max": 0.0,
@@ -960,26 +967,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 64.25,
- "completions/clipped_ratio": 0.125,
- "completions/max_length": 128.0,
- "completions/max_terminated_length": 88.0,
- "completions/mean_length": 65.125,
- "completions/mean_terminated_length": 56.142860412597656,
- "completions/min_length": 32.0,
- "completions/min_terminated_length": 32.0,
- "epoch": 0.6101694915254238,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 61.25,
+ "completions/mean_terminated_length": 61.25,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.0832659006118774,
+ "epoch": 0.2845528455284553,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.25029000639915466,
- "kl": 0.007542325067333877,
- "learning_rate": 4.2617671799875944e-07,
- "loss": -0.039806246757507324,
- "num_tokens": 134976.0,
- "reward": 0.6346843838691711,
- "reward_std": 0.019400157034397125,
- "rewards/true_env_reward_fn/mean": 0.6346843838691711,
- "rewards/true_env_reward_fn/std": 0.06388701498508453,
- "step": 36
+ "grad_norm": 0.1297609806060791,
+ "kl": 1.1829738923552213e-05,
+ "learning_rate": 7.351351351351351e-07,
+ "loss": -0.02754262089729309,
+ "num_tokens": 142361.0,
+ "reward": 0.4525124728679657,
+ "reward_std": 0.23157824575901031,
+ "rewards/true_env_reward_fn/mean": 0.4525124728679657,
+ "rewards/true_env_reward_fn/std": 0.2315782606601715,
+ "step": 35,
+ "step_time": 3.564060039998367
},
{
"clip_ratio/high_max": 0.0,
@@ -987,26 +995,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 55.75,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 66.0,
- "completions/max_terminated_length": 66.0,
- "completions/mean_length": 56.75,
- "completions/mean_terminated_length": 56.75,
- "completions/min_length": 46.0,
- "completions/min_terminated_length": 46.0,
- "epoch": 0.6271186440677966,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004802029579877853,
- "kl": 0.0034167669364251196,
- "learning_rate": 3.970106906294509e-07,
- "loss": 0.00017169890634249896,
- "num_tokens": 140050.0,
- "reward": 0.08733570575714111,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.08733570575714111,
- "rewards/true_env_reward_fn/std": 0.1842346340417862,
- "step": 37
+ "completions/max_length": 74.0,
+ "completions/max_terminated_length": 74.0,
+ "completions/mean_length": 55.125,
+ "completions/mean_terminated_length": 55.125,
+ "completions/min_length": 45.0,
+ "completions/min_terminated_length": 45.0,
+ "entropy": 1.0677781999111176,
+ "epoch": 0.2926829268292683,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.14367543160915375,
+ "kl": 1.2750254427373875e-05,
+ "learning_rate": 7.567567567567568e-07,
+ "loss": -0.001130133867263794,
+ "num_tokens": 145294.0,
+ "reward": 0.6871603727340698,
+ "reward_std": 0.2714426517486572,
+ "rewards/true_env_reward_fn/mean": 0.6871603727340698,
+ "rewards/true_env_reward_fn/std": 0.2714426517486572,
+ "step": 36,
+ "step_time": 3.6285808550001093
},
{
"clip_ratio/high_max": 0.0,
@@ -1014,26 +1023,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 56.0,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 101.0,
- "completions/max_terminated_length": 101.0,
- "completions/mean_length": 57.0,
- "completions/mean_terminated_length": 57.0,
- "completions/min_length": 31.0,
- "completions/min_terminated_length": 31.0,
- "epoch": 0.6440677966101694,
+ "completions/max_length": 104.0,
+ "completions/max_terminated_length": 104.0,
+ "completions/mean_length": 73.125,
+ "completions/mean_terminated_length": 73.125,
+ "completions/min_length": 49.0,
+ "completions/min_terminated_length": 49.0,
+ "entropy": 1.408882200717926,
+ "epoch": 0.3008130081300813,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.32230323553085327,
- "kl": 0.006699230056256056,
- "learning_rate": 3.6820641696546627e-07,
- "loss": -0.02568381279706955,
- "num_tokens": 144518.0,
- "reward": 0.28047916293144226,
- "reward_std": 0.13095833361148834,
- "rewards/true_env_reward_fn/mean": 0.28047916293144226,
- "rewards/true_env_reward_fn/std": 0.30373889207839966,
- "step": 38
+ "grad_norm": 0.13866695761680603,
+ "kl": 1.3317891898623202e-05,
+ "learning_rate": 7.783783783783784e-07,
+ "loss": 0.058712199330329895,
+ "num_tokens": 148747.0,
+ "reward": 0.638524055480957,
+ "reward_std": 0.380489706993103,
+ "rewards/true_env_reward_fn/mean": 0.638524055480957,
+ "rewards/true_env_reward_fn/std": 0.3804897367954254,
+ "step": 37,
+ "step_time": 4.57648780099953
},
{
"clip_ratio/high_max": 0.0,
@@ -1041,26 +1051,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 42.75,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 61.0,
- "completions/max_terminated_length": 61.0,
- "completions/mean_length": 43.75,
- "completions/mean_terminated_length": 43.75,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "epoch": 0.6610169491525424,
+ "completions/max_length": 81.0,
+ "completions/max_terminated_length": 81.0,
+ "completions/mean_length": 57.875,
+ "completions/mean_terminated_length": 57.875,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.3680316805839539,
+ "epoch": 0.3089430894308943,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.39349958300590515,
- "kl": 0.008502465672791004,
- "learning_rate": 3.3986507306858125e-07,
- "loss": 0.02716679871082306,
- "num_tokens": 146216.0,
- "reward": 0.7572407722473145,
- "reward_std": 0.004953920841217041,
- "rewards/true_env_reward_fn/mean": 0.7572407722473145,
- "rewards/true_env_reward_fn/std": 0.03704795241355896,
- "step": 39
+ "grad_norm": 0.13230997323989868,
+ "kl": 1.1831724805233534e-05,
+ "learning_rate": 8e-07,
+ "loss": -0.06476183235645294,
+ "num_tokens": 152794.0,
+ "reward": 0.47908467054367065,
+ "reward_std": 0.18681679666042328,
+ "rewards/true_env_reward_fn/mean": 0.47908467054367065,
+ "rewards/true_env_reward_fn/std": 0.18681679666042328,
+ "step": 38,
+ "step_time": 3.627890882000429
},
{
"clip_ratio/high_max": 0.0,
@@ -1068,26 +1079,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 60.125,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 75.0,
- "completions/max_terminated_length": 75.0,
- "completions/mean_length": 61.125,
- "completions/mean_terminated_length": 61.125,
- "completions/min_length": 40.0,
- "completions/min_terminated_length": 40.0,
- "epoch": 0.6779661016949152,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.25497403740882874,
- "kl": 0.005479680956341326,
- "learning_rate": 3.12086208942881e-07,
- "loss": -0.047570496797561646,
- "num_tokens": 149305.0,
- "reward": 0.581167995929718,
- "reward_std": 0.01358928345143795,
- "rewards/true_env_reward_fn/mean": 0.581167995929718,
- "rewards/true_env_reward_fn/std": 0.11860521137714386,
- "step": 40
+ "completions/max_length": 73.0,
+ "completions/max_terminated_length": 73.0,
+ "completions/mean_length": 56.875,
+ "completions/mean_terminated_length": 56.875,
+ "completions/min_length": 37.0,
+ "completions/min_terminated_length": 37.0,
+ "entropy": 1.3124344944953918,
+ "epoch": 0.3170731707317073,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20907950401306152,
+ "kl": 1.4425510926230345e-05,
+ "learning_rate": 7.999820918660971e-07,
+ "loss": -0.014620006084442139,
+ "num_tokens": 157337.0,
+ "reward": 0.4882892966270447,
+ "reward_std": 0.28137314319610596,
+ "rewards/true_env_reward_fn/mean": 0.4882892966270447,
+ "rewards/true_env_reward_fn/std": 0.28137317299842834,
+ "step": 39,
+ "step_time": 3.5362214279994078
},
{
"clip_ratio/high_max": 0.0,
@@ -1095,26 +1107,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 73.75,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 85.0,
- "completions/max_terminated_length": 85.0,
- "completions/mean_length": 74.75,
- "completions/mean_terminated_length": 74.75,
- "completions/min_length": 56.0,
- "completions/min_terminated_length": 56.0,
- "epoch": 0.6949152542372882,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.25451600551605225,
- "kl": 0.004066723049618304,
- "learning_rate": 2.849673988617399e-07,
- "loss": -0.0552646741271019,
- "num_tokens": 152999.0,
- "reward": 0.49457329511642456,
- "reward_std": 0.20980888605117798,
- "rewards/true_env_reward_fn/mean": 0.4945732653141022,
- "rewards/true_env_reward_fn/std": 0.28006047010421753,
- "step": 41
+ "completions/max_length": 69.0,
+ "completions/max_terminated_length": 69.0,
+ "completions/mean_length": 51.625,
+ "completions/mean_terminated_length": 51.625,
+ "completions/min_length": 35.0,
+ "completions/min_terminated_length": 35.0,
+ "entropy": 0.9928885996341705,
+ "epoch": 0.3252032520325203,
+ "frac_reward_zero_std": 1.0,
+ "grad_norm": 8.641438762424514e-05,
+ "kl": 1.1731265658454504e-05,
+ "learning_rate": 7.99928369067895e-07,
+ "loss": 5.910313234380737e-07,
+ "num_tokens": 160166.0,
+ "reward": 0.6114685535430908,
+ "reward_std": 0.1678776890039444,
+ "rewards/true_env_reward_fn/mean": 0.6114685535430908,
+ "rewards/true_env_reward_fn/std": 0.1678776890039444,
+ "step": 40,
+ "step_time": 3.1957039770022675
},
{
"clip_ratio/high_max": 0.0,
@@ -1122,26 +1135,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 50.875,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 86.0,
- "completions/max_terminated_length": 86.0,
- "completions/mean_length": 51.875,
- "completions/mean_terminated_length": 51.875,
- "completions/min_length": 37.0,
- "completions/min_terminated_length": 37.0,
- "epoch": 0.711864406779661,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.27449387311935425,
- "kl": 0.0040840215515345335,
- "learning_rate": 2.5860389863462763e-07,
- "loss": 0.09527499228715897,
- "num_tokens": 157978.0,
- "reward": 0.25437501072883606,
- "reward_std": 0.12575000524520874,
- "rewards/true_env_reward_fn/mean": 0.25437501072883606,
- "rewards/true_env_reward_fn/std": 0.2772188186645508,
- "step": 42
+ "completions/max_length": 76.0,
+ "completions/max_terminated_length": 76.0,
+ "completions/mean_length": 54.75,
+ "completions/mean_terminated_length": 54.75,
+ "completions/min_length": 33.0,
+ "completions/min_terminated_length": 33.0,
+ "entropy": 1.2997828722000122,
+ "epoch": 0.3333333333333333,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.21271590888500214,
+ "kl": 1.3209032658778597e-05,
+ "learning_rate": 7.99838836415769e-07,
+ "loss": 0.033298641443252563,
+ "num_tokens": 165884.0,
+ "reward": 0.2860966920852661,
+ "reward_std": 0.2721884846687317,
+ "rewards/true_env_reward_fn/mean": 0.2860966920852661,
+ "rewards/true_env_reward_fn/std": 0.2721884846687317,
+ "step": 41,
+ "step_time": 3.6851942720004445
},
{
"clip_ratio/high_max": 0.0,
@@ -1149,26 +1163,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 45.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 89.0,
- "completions/max_terminated_length": 89.0,
- "completions/mean_length": 46.625,
- "completions/mean_terminated_length": 46.625,
- "completions/min_length": 25.0,
- "completions/min_terminated_length": 25.0,
- "epoch": 0.7288135593220338,
+ "completions/max_length": 70.0,
+ "completions/max_terminated_length": 70.0,
+ "completions/mean_length": 48.875,
+ "completions/mean_terminated_length": 48.875,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.4380556344985962,
+ "epoch": 0.34146341463414637,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.36372485756874084,
- "kl": 0.00767878710757941,
- "learning_rate": 2.3308831101760483e-07,
- "loss": -0.0198854748159647,
- "num_tokens": 160859.0,
- "reward": 0.5616356730461121,
- "reward_std": 0.07793298363685608,
- "rewards/true_env_reward_fn/mean": 0.5616356730461121,
- "rewards/true_env_reward_fn/std": 0.16229310631752014,
- "step": 43
+ "grad_norm": 0.17314757406711578,
+ "kl": 9.354779194836738e-06,
+ "learning_rate": 7.997135019265325e-07,
+ "loss": 0.08398272097110748,
+ "num_tokens": 172067.0,
+ "reward": -0.003943998366594315,
+ "reward_std": 0.13122709095478058,
+ "rewards/true_env_reward_fn/mean": -0.003943998366594315,
+ "rewards/true_env_reward_fn/std": 0.13122709095478058,
+ "step": 42,
+ "step_time": 3.545334507000007
},
{
"clip_ratio/high_max": 0.0,
@@ -1176,26 +1191,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 72.125,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 118.0,
- "completions/max_terminated_length": 118.0,
- "completions/mean_length": 73.125,
- "completions/mean_terminated_length": 73.125,
- "completions/min_length": 28.0,
- "completions/min_terminated_length": 28.0,
- "epoch": 0.7457627118644068,
+ "completions/max_length": 123.0,
+ "completions/max_terminated_length": 123.0,
+ "completions/mean_length": 71.0,
+ "completions/mean_terminated_length": 71.0,
+ "completions/min_length": 43.0,
+ "completions/min_terminated_length": 43.0,
+ "entropy": 1.163844645023346,
+ "epoch": 0.34959349593495936,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.33719322085380554,
- "kl": 0.006818318506702781,
- "learning_rate": 2.0851026044276405e-07,
- "loss": 0.0655989721417427,
- "num_tokens": 163440.0,
- "reward": 0.5270739793777466,
- "reward_std": 0.10828020423650742,
- "rewards/true_env_reward_fn/mean": 0.5270739793777466,
- "rewards/true_env_reward_fn/std": 0.1423584520816803,
- "step": 44
+ "grad_norm": 0.15747681260108948,
+ "kl": 1.3550960375141585e-05,
+ "learning_rate": 7.995523768227198e-07,
+ "loss": 0.05901219695806503,
+ "num_tokens": 176427.0,
+ "reward": 0.3297747075557709,
+ "reward_std": 0.4647028148174286,
+ "rewards/true_env_reward_fn/mean": 0.3297747075557709,
+ "rewards/true_env_reward_fn/std": 0.464702844619751,
+ "step": 43,
+ "step_time": 5.4708715960005065
},
{
"clip_ratio/high_max": 0.0,
@@ -1203,26 +1219,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 72.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 107.0,
- "completions/max_terminated_length": 107.0,
- "completions/mean_length": 73.5,
- "completions/mean_terminated_length": 73.5,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "epoch": 0.7627118644067796,
- "frac_reward_zero_std": 1.0,
- "grad_norm": 0.004031375516206026,
- "kl": 0.0033984831534326077,
- "learning_rate": 1.849560782091445e-07,
- "loss": 0.00016979672363959253,
- "num_tokens": 167260.0,
- "reward": 0.5474250316619873,
- "reward_std": 0.0,
- "rewards/true_env_reward_fn/mean": 0.5474250316619873,
- "rewards/true_env_reward_fn/std": 0.03466380015015602,
- "step": 45
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 57.625,
+ "completions/mean_terminated_length": 57.625,
+ "completions/min_length": 42.0,
+ "completions/min_terminated_length": 42.0,
+ "entropy": 1.3323996663093567,
+ "epoch": 0.35772357723577236,
+ "frac_reward_zero_std": 0.5,
+ "grad_norm": 0.15227818489074707,
+ "kl": 1.1237668786634458e-05,
+ "learning_rate": 7.993554755315805e-07,
+ "loss": 0.0660967156291008,
+ "num_tokens": 181912.0,
+ "reward": 0.22226500511169434,
+ "reward_std": 0.2765512466430664,
+ "rewards/true_env_reward_fn/mean": 0.22226500511169434,
+ "rewards/true_env_reward_fn/std": 0.2765512466430664,
+ "step": 44,
+ "step_time": 3.940563359999942
},
{
"clip_ratio/high_max": 0.0,
@@ -1230,26 +1247,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 58.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 80.0,
- "completions/max_terminated_length": 80.0,
- "completions/mean_length": 59.625,
- "completions/mean_terminated_length": 59.625,
- "completions/min_length": 35.0,
- "completions/min_terminated_length": 35.0,
- "epoch": 0.7796610169491526,
+ "completions/max_length": 87.0,
+ "completions/max_terminated_length": 87.0,
+ "completions/mean_length": 56.375,
+ "completions/mean_terminated_length": 56.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.241302490234375,
+ "epoch": 0.36585365853658536,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.31983038783073425,
- "kl": 0.005334719025995582,
- "learning_rate": 1.6250849924089482e-07,
- "loss": 0.04526376724243164,
- "num_tokens": 171845.0,
- "reward": 0.2948041558265686,
- "reward_std": 0.0032749995589256287,
- "rewards/true_env_reward_fn/mean": 0.2948041558265686,
- "rewards/true_env_reward_fn/std": 0.16418558359146118,
- "step": 46
+ "grad_norm": 0.1711702048778534,
+ "kl": 1.1479866316221887e-05,
+ "learning_rate": 7.991228156837879e-07,
+ "loss": 0.0959811806678772,
+ "num_tokens": 186099.0,
+ "reward": 0.4569639563560486,
+ "reward_std": 0.356449693441391,
+ "rewards/true_env_reward_fn/mean": 0.4569639563560486,
+ "rewards/true_env_reward_fn/std": 0.356449693441391,
+ "step": 45,
+ "step_time": 3.947248132999448
},
{
"clip_ratio/high_max": 0.0,
@@ -1257,26 +1275,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 55.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 94.0,
- "completions/max_terminated_length": 94.0,
- "completions/mean_length": 56.625,
- "completions/mean_terminated_length": 56.625,
- "completions/min_length": 29.0,
- "completions/min_terminated_length": 29.0,
- "epoch": 0.7966101694915254,
- "frac_reward_zero_std": 0.5,
- "grad_norm": 0.27713826298713684,
- "kl": 0.007050895364955068,
- "learning_rate": 1.412463714778343e-07,
- "loss": -0.02804308384656906,
- "num_tokens": 174794.0,
- "reward": 0.4624658226966858,
- "reward_std": 0.03971340134739876,
- "rewards/true_env_reward_fn/mean": 0.4624658226966858,
- "rewards/true_env_reward_fn/std": 0.10674785077571869,
- "step": 47
+ "completions/max_length": 83.0,
+ "completions/max_terminated_length": 83.0,
+ "completions/mean_length": 66.625,
+ "completions/mean_terminated_length": 66.625,
+ "completions/min_length": 46.0,
+ "completions/min_terminated_length": 46.0,
+ "entropy": 1.5153677463531494,
+ "epoch": 0.37398373983739835,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.22329360246658325,
+ "kl": 1.3615897842100821e-05,
+ "learning_rate": 7.988544181118608e-07,
+ "loss": 0.07407481223344803,
+ "num_tokens": 192056.0,
+ "reward": 0.2950569987297058,
+ "reward_std": 0.2872281074523926,
+ "rewards/true_env_reward_fn/mean": 0.2950569987297058,
+ "rewards/true_env_reward_fn/std": 0.28722813725471497,
+ "step": 46,
+ "step_time": 4.1211709569997765
},
{
"clip_ratio/high_max": 0.0,
@@ -1284,26 +1303,27 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 43.375,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 63.0,
- "completions/max_terminated_length": 63.0,
- "completions/mean_length": 44.375,
- "completions/mean_terminated_length": 44.375,
- "completions/min_length": 26.0,
- "completions/min_terminated_length": 26.0,
- "epoch": 0.8135593220338984,
+ "completions/max_length": 62.0,
+ "completions/max_terminated_length": 62.0,
+ "completions/mean_length": 54.5,
+ "completions/mean_terminated_length": 54.5,
+ "completions/min_length": 44.0,
+ "completions/min_terminated_length": 44.0,
+ "entropy": 1.105223298072815,
+ "epoch": 0.3821138211382114,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.3878757357597351,
- "kl": 0.009432611055672169,
- "learning_rate": 1.2124437891918993e-07,
- "loss": 0.10240262746810913,
- "num_tokens": 178781.0,
- "reward": 0.4739072918891907,
- "reward_std": 0.13465046882629395,
- "rewards/true_env_reward_fn/mean": 0.4739072918891907,
- "rewards/true_env_reward_fn/std": 0.38713982701301575,
- "step": 48
+ "grad_norm": 0.1306377500295639,
+ "kl": 1.2826244528696407e-05,
+ "learning_rate": 7.985503068482974e-07,
+ "loss": 0.014609627425670624,
+ "num_tokens": 195544.0,
+ "reward": 0.5289265513420105,
+ "reward_std": 0.3883950710296631,
+ "rewards/true_env_reward_fn/mean": 0.5289265513420105,
+ "rewards/true_env_reward_fn/std": 0.3883951008319855,
+ "step": 47,
+ "step_time": 2.938600743000279
},
{
"clip_ratio/high_max": 0.0,
@@ -1311,26 +1331,55 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 56.625,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 96.0,
- "completions/max_terminated_length": 96.0,
- "completions/mean_length": 57.625,
- "completions/mean_terminated_length": 57.625,
- "completions/min_length": 41.0,
- "completions/min_terminated_length": 41.0,
- "epoch": 0.8305084745762712,
+ "completions/max_length": 116.0,
+ "completions/max_terminated_length": 116.0,
+ "completions/mean_length": 67.375,
+ "completions/mean_terminated_length": 67.375,
+ "completions/min_length": 39.0,
+ "completions/min_terminated_length": 39.0,
+ "entropy": 1.5243317484855652,
+ "epoch": 0.3902439024390244,
+ "frac_reward_zero_std": 0.0,
+ "grad_norm": 0.20161111652851105,
+ "kl": 1.4497059055429418e-05,
+ "learning_rate": 7.982105091234235e-07,
+ "loss": 0.23342597484588623,
+ "num_tokens": 198691.0,
+ "reward": 0.45001715421676636,
+ "reward_std": 0.2565726041793823,
+ "rewards/true_env_reward_fn/mean": 0.45001715421676636,
+ "rewards/true_env_reward_fn/std": 0.2565726041793823,
+ "step": 48,
+ "step_time": 4.91795033499875
+ },
+ {
+ "clip_ratio/high_max": 0.0,
+ "clip_ratio/high_mean": 0.0,
+ "clip_ratio/low_mean": 0.0,
+ "clip_ratio/low_min": 0.0,
+ "clip_ratio/region_mean": 0.0,
+ "completions/clipped_ratio": 0.0,
+ "completions/max_length": 79.0,
+ "completions/max_terminated_length": 79.0,
+ "completions/mean_length": 55.375,
+ "completions/mean_terminated_length": 55.375,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.2544435858726501,
+ "epoch": 0.3983739837398374,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.2935178279876709,
- "kl": 0.012418681057170033,
- "learning_rate": 1.0257277929332331e-07,
- "loss": -0.026224076747894287,
- "num_tokens": 182126.0,
- "reward": 0.4184165894985199,
- "reward_std": 0.15121765434741974,
- "rewards/true_env_reward_fn/mean": 0.4184166193008423,
- "rewards/true_env_reward_fn/std": 0.32535234093666077,
- "step": 49
+ "grad_norm": 0.11793916672468185,
+ "kl": 1.3676196886081016e-05,
+ "learning_rate": 7.978350553629554e-07,
+ "loss": -0.016418367624282837,
+ "num_tokens": 202994.0,
+ "reward": 0.4054500162601471,
+ "reward_std": 0.20634961128234863,
+ "rewards/true_env_reward_fn/mean": 0.4054500162601471,
+ "rewards/true_env_reward_fn/std": 0.20634961128234863,
+ "step": 49,
+ "step_time": 3.626596234000317
},
{
"clip_ratio/high_max": 0.0,
@@ -1338,32 +1387,33 @@
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
- "completion_length": 40.5,
"completions/clipped_ratio": 0.0,
- "completions/max_length": 60.0,
- "completions/max_terminated_length": 60.0,
- "completions/mean_length": 41.5,
- "completions/mean_terminated_length": 41.5,
- "completions/min_length": 19.0,
- "completions/min_terminated_length": 19.0,
- "epoch": 0.847457627118644,
+ "completions/max_length": 91.0,
+ "completions/max_terminated_length": 91.0,
+ "completions/mean_length": 62.5,
+ "completions/mean_terminated_length": 62.5,
+ "completions/min_length": 36.0,
+ "completions/min_terminated_length": 36.0,
+ "entropy": 1.022342562675476,
+ "epoch": 0.4065040650406504,
"frac_reward_zero_std": 0.5,
- "grad_norm": 0.39011478424072266,
- "kl": 0.009452722501009703,
- "learning_rate": 8.529715727489912e-08,
- "loss": -0.050017084926366806,
- "num_tokens": 183790.0,
- "reward": 0.7530088424682617,
- "reward_std": 0.003693848382681608,
- "rewards/true_env_reward_fn/mean": 0.7530088424682617,
- "rewards/true_env_reward_fn/std": 0.11074119806289673,
- "step": 50
+ "grad_norm": 0.16596083343029022,
+ "kl": 1.1194244052603608e-05,
+ "learning_rate": 7.974239791852739e-07,
+ "loss": 0.0499756895005703,
+ "num_tokens": 205770.0,
+ "reward": 0.5639185309410095,
+ "reward_std": 0.1721728891134262,
+ "rewards/true_env_reward_fn/mean": 0.5639185309410095,
+ "rewards/true_env_reward_fn/std": 0.1721728891134262,
+ "step": 50,
+ "step_time": 3.9679293660010444
}
],
"logging_steps": 1,
- "max_steps": 59,
- "num_input_tokens_seen": 183790,
- "num_train_epochs": 1,
+ "max_steps": 369,
+ "num_input_tokens_seen": 205770,
+ "num_train_epochs": 3,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
@@ -1378,7 +1428,7 @@
}
},
"total_flos": 0.0,
- "train_batch_size": 2,
+ "train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
diff --git a/checkpoint-50/training_args.bin b/checkpoint-50/training_args.bin
index f8d165bfd5cb3b010587480bfcbec8d95179d8f8..8d94c3c38f17faf8a60976b504514708acad4864 100644
--- a/checkpoint-50/training_args.bin
+++ b/checkpoint-50/training_args.bin
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:ea770df1d3992e7a68c3cb36aaa8be672d8ac50672d965ab8faafe915ee61195
-size 6673
+oid sha256:fe7f1c61e0e89afd793570a9a89dda9ed6569838bac7b1e7d383c47f3e040774
+size 6776
diff --git a/ref/adapter_config.json b/ref/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1335f363e822e5c942fa1ace6e0eb5fabdd99524
--- /dev/null
+++ b/ref/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "unsloth/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "lora_ga_config": null,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.19.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_bdlora": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/ref/adapter_model.safetensors b/ref/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1748ecc34d0d4aae1e8bc8135cb16bc901705fd4
--- /dev/null
+++ b/ref/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b962b0084aec460781aac28e8d34bea11fb9022883ecd77704b8455ce2d723a2
+size 8731128
diff --git a/tokenizer.json b/tokenizer.json
index af979ec5282fda15d3de42a1719f7aff8f802448..34510ff0037cd50428af467a17ead5a96140a32c 100644
--- a/tokenizer.json
+++ b/tokenizer.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:1e2928ca692824ff18793d8aafde64d8320497b91445e558619e994c04e67c5d
-size 11422520
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892
diff --git a/tokenizer_config.json b/tokenizer_config.json
index e36b2aab7459091b366adc1d6f1477527292f961..9aa5955b7dd3b15995a880b534c69ad538218d2d 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -5,197 +5,12 @@
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
- "is_local": false,
+ "is_local": true,
+ "local_files_only": false,
"model_max_length": 32768,
"pad_token": "<|im_end|>",
"padding_side": "left",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
- "unk_token": null,
- "added_tokens_decoder": {
- "151643": {
- "content": "<|endoftext|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151644": {
- "content": "<|im_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151645": {
- "content": "<|im_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151646": {
- "content": "<|object_ref_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151647": {
- "content": "<|object_ref_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151648": {
- "content": "<|box_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151649": {
- "content": "<|box_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151650": {
- "content": "<|quad_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151651": {
- "content": "<|quad_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151652": {
- "content": "<|vision_start|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151653": {
- "content": "<|vision_end|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151654": {
- "content": "<|vision_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151655": {
- "content": "<|image_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151656": {
- "content": "<|video_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- "151657": {
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151658": {
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151659": {
- "content": "<|fim_prefix|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151660": {
- "content": "<|fim_middle|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151661": {
- "content": "<|fim_suffix|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151662": {
- "content": "<|fim_pad|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151663": {
- "content": "<|repo_name|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151664": {
- "content": "<|file_sep|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": false
- },
- "151665": {
- "content": "<|PAD_TOKEN|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- }
- }
+ "unk_token": null
}
diff --git a/training_metadata.json b/training_metadata.json
index 63129a35ee07e418ee6d1198cd0bc22322b5c1d4..e58bec5fe7747648f889ae244ee4fdd65dad28fc 100644
--- a/training_metadata.json
+++ b/training_metadata.json
@@ -1,6 +1,6 @@
{
"sft_checkpoint": "./sft_checkpoint",
- "rollouts_per_difficulty": 8,
+ "rollouts_per_difficulty": 16,
"difficulty_mix": [
1,
1,
@@ -8,15 +8,15 @@
2,
3
],
- "num_train_epochs": 1.0,
- "per_device_batch": 2,
- "grad_accum": 4,
+ "num_train_epochs": 3.0,
+ "per_device_batch": 4,
+ "grad_accum": 2,
"num_generations": 4,
- "lr": 1e-06,
+ "lr": 8e-07,
"beta": 0.05,
- "gamma": 0.95,
+ "gamma": 0.98,
"seed": 42,
- "n_unique_prompts": 118,
- "n_state_snapshots": 271,
+ "n_unique_prompts": 247,
+ "n_state_snapshots": 682,
"use_vllm": false
}
\ No newline at end of file