Hugging Face's logo

Spaces:
akhiilll
/
claims-env-pro
Sleeping

App Files Community
akhiilll commited on
Commit
4c6c4db
·
verified ·
1 Parent(s): e89d325

GRPO run: grpo-20260426-062846

Browse files
Files changed (29) hide show
  1. .gitattributes +3 -0
  2. runs/grpo-20260426-062846/before_after.png +0 -0
  3. runs/grpo-20260426-062846/grpo_training.png +3 -0
  4. runs/grpo-20260426-062846/lora-adapter/README.md +209 -0
  5. runs/grpo-20260426-062846/lora-adapter/adapter_config.json +48 -0
  6. runs/grpo-20260426-062846/lora-adapter/adapter_model.safetensors +3 -0
  7. runs/grpo-20260426-062846/lora-adapter/added_tokens.json +24 -0
  8. runs/grpo-20260426-062846/lora-adapter/merges.txt +0 -0
  9. runs/grpo-20260426-062846/lora-adapter/special_tokens_map.json +31 -0
  10. runs/grpo-20260426-062846/lora-adapter/tokenizer.json +3 -0
  11. runs/grpo-20260426-062846/lora-adapter/tokenizer_config.json +208 -0
  12. runs/grpo-20260426-062846/lora-adapter/vocab.json +0 -0
  13. runs/grpo-20260426-062846/metrics.json +38 -0
  14. runs/grpo-20260426-062846/metrics.jsonl +81 -0
  15. runs/grpo-20260426-062846/per_task_breakdown.json +83 -0
  16. runs/grpo-20260426-062846/trainer/checkpoint-80/README.md +209 -0
  17. runs/grpo-20260426-062846/trainer/checkpoint-80/adapter_config.json +48 -0
  18. runs/grpo-20260426-062846/trainer/checkpoint-80/adapter_model.safetensors +3 -0
  19. runs/grpo-20260426-062846/trainer/checkpoint-80/added_tokens.json +24 -0
  20. runs/grpo-20260426-062846/trainer/checkpoint-80/merges.txt +0 -0
  21. runs/grpo-20260426-062846/trainer/checkpoint-80/optimizer.pt +3 -0
  22. runs/grpo-20260426-062846/trainer/checkpoint-80/rng_state.pth +3 -0
  23. runs/grpo-20260426-062846/trainer/checkpoint-80/scheduler.pt +3 -0
  24. runs/grpo-20260426-062846/trainer/checkpoint-80/special_tokens_map.json +31 -0
  25. runs/grpo-20260426-062846/trainer/checkpoint-80/tokenizer.json +3 -0
  26. runs/grpo-20260426-062846/trainer/checkpoint-80/tokenizer_config.json +208 -0
  27. runs/grpo-20260426-062846/trainer/checkpoint-80/trainer_state.json +1073 -0
  28. runs/grpo-20260426-062846/trainer/checkpoint-80/training_args.bin +3 -0
  29. runs/grpo-20260426-062846/trainer/checkpoint-80/vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ runs/grpo-20260426-062846/grpo_training.png filter=lfs diff=lfs merge=lfs -text
37
+ runs/grpo-20260426-062846/lora-adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ runs/grpo-20260426-062846/trainer/checkpoint-80/tokenizer.json filter=lfs diff=lfs merge=lfs -text
runs/grpo-20260426-062846/before_after.png ADDED
runs/grpo-20260426-062846/grpo_training.png ADDED

Git LFS Details

  • SHA256: 34f8c1e88c33298fea75f31dbecfeb788f468b71837483c71405e6cc1379a7a5
  • Pointer size: 131 Bytes
  • Size of remote file: 278 kB
runs/grpo-20260426-062846/lora-adapter/README.md ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen2.5-1.5B-Instruct
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
7
+ - grpo
8
+ - lora
9
+ - transformers
10
+ - trl
11
+ ---
12
+
13
+ # Model Card for Model ID
14
+
15
+ <!-- Provide a quick summary of what the model is/does. -->
16
+
17
+
18
+
19
+ ## Model Details
20
+
21
+ ### Model Description
22
+
23
+ <!-- Provide a longer summary of what this model is. -->
24
+
25
+
26
+
27
+ - **Developed by:** [More Information Needed]
28
+ - **Funded by [optional]:** [More Information Needed]
29
+ - **Shared by [optional]:** [More Information Needed]
30
+ - **Model type:** [More Information Needed]
31
+ - **Language(s) (NLP):** [More Information Needed]
32
+ - **License:** [More Information Needed]
33
+ - **Finetuned from model [optional]:** [More Information Needed]
34
+
35
+ ### Model Sources [optional]
36
+
37
+ <!-- Provide the basic links for the model. -->
38
+
39
+ - **Repository:** [More Information Needed]
40
+ - **Paper [optional]:** [More Information Needed]
41
+ - **Demo [optional]:** [More Information Needed]
42
+
43
+ ## Uses
44
+
45
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
46
+
47
+ ### Direct Use
48
+
49
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
50
+
51
+ [More Information Needed]
52
+
53
+ ### Downstream Use [optional]
54
+
55
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
56
+
57
+ [More Information Needed]
58
+
59
+ ### Out-of-Scope Use
60
+
61
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
62
+
63
+ [More Information Needed]
64
+
65
+ ## Bias, Risks, and Limitations
66
+
67
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
68
+
69
+ [More Information Needed]
70
+
71
+ ### Recommendations
72
+
73
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
74
+
75
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
76
+
77
+ ## How to Get Started with the Model
78
+
79
+ Use the code below to get started with the model.
80
+
81
+ [More Information Needed]
82
+
83
+ ## Training Details
84
+
85
+ ### Training Data
86
+
87
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
88
+
89
+ [More Information Needed]
90
+
91
+ ### Training Procedure
92
+
93
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
94
+
95
+ #### Preprocessing [optional]
96
+
97
+ [More Information Needed]
98
+
99
+
100
+ #### Training Hyperparameters
101
+
102
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
103
+
104
+ #### Speeds, Sizes, Times [optional]
105
+
106
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
107
+
108
+ [More Information Needed]
109
+
110
+ ## Evaluation
111
+
112
+ <!-- This section describes the evaluation protocols and provides the results. -->
113
+
114
+ ### Testing Data, Factors & Metrics
115
+
116
+ #### Testing Data
117
+
118
+ <!-- This should link to a Dataset Card if possible. -->
119
+
120
+ [More Information Needed]
121
+
122
+ #### Factors
123
+
124
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
125
+
126
+ [More Information Needed]
127
+
128
+ #### Metrics
129
+
130
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
131
+
132
+ [More Information Needed]
133
+
134
+ ### Results
135
+
136
+ [More Information Needed]
137
+
138
+ #### Summary
139
+
140
+
141
+
142
+ ## Model Examination [optional]
143
+
144
+ <!-- Relevant interpretability work for the model goes here -->
145
+
146
+ [More Information Needed]
147
+
148
+ ## Environmental Impact
149
+
150
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
151
+
152
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
153
+
154
+ - **Hardware Type:** [More Information Needed]
155
+ - **Hours used:** [More Information Needed]
156
+ - **Cloud Provider:** [More Information Needed]
157
+ - **Compute Region:** [More Information Needed]
158
+ - **Carbon Emitted:** [More Information Needed]
159
+
160
+ ## Technical Specifications [optional]
161
+
162
+ ### Model Architecture and Objective
163
+
164
+ [More Information Needed]
165
+
166
+ ### Compute Infrastructure
167
+
168
+ [More Information Needed]
169
+
170
+ #### Hardware
171
+
172
+ [More Information Needed]
173
+
174
+ #### Software
175
+
176
+ [More Information Needed]
177
+
178
+ ## Citation [optional]
179
+
180
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
181
+
182
+ **BibTeX:**
183
+
184
+ [More Information Needed]
185
+
186
+ **APA:**
187
+
188
+ [More Information Needed]
189
+
190
+ ## Glossary [optional]
191
+
192
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
193
+
194
+ [More Information Needed]
195
+
196
+ ## More Information [optional]
197
+
198
+ [More Information Needed]
199
+
200
+ ## Model Card Authors [optional]
201
+
202
+ [More Information Needed]
203
+
204
+ ## Model Card Contact
205
+
206
+ [More Information Needed]
207
+ ### Framework versions
208
+
209
+ - PEFT 0.19.1
runs/grpo-20260426-062846/lora-adapter/adapter_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 32,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 16,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "down_proj",
34
+ "o_proj",
35
+ "gate_proj",
36
+ "v_proj",
37
+ "k_proj",
38
+ "up_proj",
39
+ "q_proj"
40
+ ],
41
+ "target_parameters": null,
42
+ "task_type": "CAUSAL_LM",
43
+ "trainable_token_indices": null,
44
+ "use_bdlora": null,
45
+ "use_dora": false,
46
+ "use_qalora": false,
47
+ "use_rslora": false
48
+ }
runs/grpo-20260426-062846/lora-adapter/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a7c44d68cc79ca09273589fe9c8409651a225d6bdcafe25a1d674df4c1bb199
3
+ size 73911112
runs/grpo-20260426-062846/lora-adapter/added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|box_end|>": 151649,
5
+ "<|box_start|>": 151648,
6
+ "<|endoftext|>": 151643,
7
+ "<|file_sep|>": 151664,
8
+ "<|fim_middle|>": 151660,
9
+ "<|fim_pad|>": 151662,
10
+ "<|fim_prefix|>": 151659,
11
+ "<|fim_suffix|>": 151661,
12
+ "<|im_end|>": 151645,
13
+ "<|im_start|>": 151644,
14
+ "<|image_pad|>": 151655,
15
+ "<|object_ref_end|>": 151647,
16
+ "<|object_ref_start|>": 151646,
17
+ "<|quad_end|>": 151651,
18
+ "<|quad_start|>": 151650,
19
+ "<|repo_name|>": 151663,
20
+ "<|video_pad|>": 151656,
21
+ "<|vision_end|>": 151653,
22
+ "<|vision_pad|>": 151654,
23
+ "<|vision_start|>": 151652
24
+ }
runs/grpo-20260426-062846/lora-adapter/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
runs/grpo-20260426-062846/lora-adapter/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
runs/grpo-20260426-062846/lora-adapter/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
3
+ size 11422063
runs/grpo-20260426-062846/lora-adapter/tokenizer_config.json ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ }
181
+ },
182
+ "additional_special_tokens": [
183
+ "<|im_start|>",
184
+ "<|im_end|>",
185
+ "<|object_ref_start|>",
186
+ "<|object_ref_end|>",
187
+ "<|box_start|>",
188
+ "<|box_end|>",
189
+ "<|quad_start|>",
190
+ "<|quad_end|>",
191
+ "<|vision_start|>",
192
+ "<|vision_end|>",
193
+ "<|vision_pad|>",
194
+ "<|image_pad|>",
195
+ "<|video_pad|>"
196
+ ],
197
+ "bos_token": null,
198
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
199
+ "clean_up_tokenization_spaces": false,
200
+ "eos_token": "<|im_end|>",
201
+ "errors": "replace",
202
+ "extra_special_tokens": {},
203
+ "model_max_length": 131072,
204
+ "pad_token": "<|endoftext|>",
205
+ "split_special_tokens": false,
206
+ "tokenizer_class": "Qwen2Tokenizer",
207
+ "unk_token": null
208
+ }
runs/grpo-20260426-062846/lora-adapter/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/grpo-20260426-062846/metrics.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_id": "grpo-20260426-062846",
3
+ "base_model": "Qwen/Qwen2.5-1.5B-Instruct",
4
+ "trainer": "trl.GRPOTrainer",
5
+ "num_steps": 80,
6
+ "num_generations": 4,
7
+ "batch_size": 2,
8
+ "grad_accum": 2,
9
+ "learning_rate": 5e-06,
10
+ "case_repeats": 8,
11
+ "dataset_rows": 40,
12
+ "reward_functions": [
13
+ "format_reward_fn",
14
+ "env_reward_fn"
15
+ ],
16
+ "ws_url": "wss://akhiilll-claims-env-pro.hf.space/ws",
17
+ "env_url": "https://akhiilll-claims-env-pro.hf.space",
18
+ "dense_costs": {
19
+ "VIEW_QUEUE": -0.1,
20
+ "OPEN_CLAIM": -0.1,
21
+ "REVIEW_DOCUMENTS": -0.2,
22
+ "CHECK_POLICY": -0.1,
23
+ "INVESTIGATE_FRAUD": -0.3,
24
+ "REQUEST_INFO": -0.5,
25
+ "VERIFY_PURCHASE": -0.3,
26
+ "HANDLE_APPEAL": -0.1,
27
+ "APPROVE": 0.0,
28
+ "DENY": 0.0,
29
+ "ESCALATE": -0.2,
30
+ "END_SHIFT": 0.0
31
+ },
32
+ "plaid_discrepancy_bonus": 2.0,
33
+ "first_reward": 0.4336448833346367,
34
+ "last_reward": -3.0,
35
+ "best_reward": 0.5562500059604645,
36
+ "worst_reward": -3.0,
37
+ "mean_reward": -2.2041496865218506
38
+ }
runs/grpo-20260426-062846/metrics.jsonl ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"loss": 0.0, "grad_norm": 0.706524133682251, "learning_rate": 6.25e-07, "completion_length": 43.6875, "rewards/format_reward_fn": 0.59375, "rewards/env_reward_fn": -0.16010507941246033, "reward": 0.4336448833346367, "reward_std": 1.0901877880096436, "kl": 0.0, "epoch": 0.1, "step": 1}
2
+ {"loss": -0.0, "grad_norm": 0.46928897500038147, "learning_rate": 1.25e-06, "completion_length": 148.6875, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.479765623807907, "reward": -1.5735156387090683, "reward_std": 0.6494516134262085, "kl": 0.0, "epoch": 0.2, "step": 2}
3
+ {"loss": 0.0001, "grad_norm": 0.050346001982688904, "learning_rate": 1.8750000000000003e-06, "completion_length": 241.1875, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0014371701690834016, "epoch": 0.3, "step": 3}
4
+ {"loss": 0.0001, "grad_norm": 0.06192959100008011, "learning_rate": 2.5e-06, "completion_length": 240.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002273249381687492, "epoch": 0.4, "step": 4}
5
+ {"loss": 0.0, "grad_norm": 0.001998545601963997, "learning_rate": 3.125e-06, "completion_length": 256.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0003419504646444693, "epoch": 0.5, "step": 5}
6
+ {"loss": 0.0, "grad_norm": 0.3350253701210022, "learning_rate": 3.7500000000000005e-06, "completion_length": 121.75, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.1962499916553497, "reward": -1.2899999916553497, "reward_std": 0.3629912734031677, "kl": 0.001182895473903045, "epoch": 0.6, "step": 6}
7
+ {"loss": 0.0, "grad_norm": 0.616795539855957, "learning_rate": 4.3750000000000005e-06, "completion_length": 105.4375, "rewards/format_reward_fn": -0.125, "rewards/env_reward_fn": -1.271875023841858, "reward": -1.3968750089406967, "reward_std": 0.2832387089729309, "kl": 0.0011062583071179688, "epoch": 0.7, "step": 7}
8
+ {"loss": 0.0, "grad_norm": 0.4317721724510193, "learning_rate": 5e-06, "completion_length": 197.875, "rewards/format_reward_fn": -0.875, "rewards/env_reward_fn": -1.7950000166893005, "reward": -2.6700000762939453, "reward_std": 0.6599999666213989, "kl": 0.0011472782352939248, "epoch": 0.8, "step": 8}
9
+ {"loss": 0.0, "grad_norm": 0.5284587740898132, "learning_rate": 4.997620553954645e-06, "completion_length": 149.125, "rewards/format_reward_fn": -0.03125, "rewards/env_reward_fn": -1.2937500476837158, "reward": -1.3249999564141035, "reward_std": 0.5629094392061234, "kl": 0.0006621622887905687, "epoch": 0.9, "step": 9}
10
+ {"loss": 0.0, "grad_norm": 0.3924754858016968, "learning_rate": 4.990486745229364e-06, "completion_length": 158.25, "rewards/format_reward_fn": -0.0625, "rewards/env_reward_fn": -1.2162500023841858, "reward": -1.2787500023841858, "reward_std": 0.2405785173177719, "kl": 0.0005572861700784415, "epoch": 1.0, "step": 10}
11
+ {"loss": 0.0, "grad_norm": 0.011886635795235634, "learning_rate": 4.978612153434527e-06, "completion_length": 226.25, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0009066301281563938, "epoch": 1.1, "step": 11}
12
+ {"loss": 0.0, "grad_norm": 0.00472637265920639, "learning_rate": 4.962019382530521e-06, "completion_length": 229.4375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0004253629158483818, "epoch": 1.2, "step": 12}
13
+ {"loss": 0.0, "grad_norm": 0.623696506023407, "learning_rate": 4.9407400177998335e-06, "completion_length": 132.75, "rewards/format_reward_fn": 0.0, "rewards/env_reward_fn": -1.4375000596046448, "reward": -1.4374999403953552, "reward_std": 0.8298223614692688, "kl": 0.0006600431515835226, "epoch": 1.3, "step": 13}
14
+ {"loss": 0.0, "grad_norm": 0.398609459400177, "learning_rate": 4.914814565722671e-06, "completion_length": 134.375, "rewards/format_reward_fn": -0.1875, "rewards/env_reward_fn": -1.3949999809265137, "reward": -1.5824999958276749, "reward_std": 0.536240816116333, "kl": 0.000562123314011842, "epoch": 1.4, "step": 14}
15
+ {"loss": 0.0, "grad_norm": 0.009604142978787422, "learning_rate": 4.884292376870567e-06, "completion_length": 240.5, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0008011562749743462, "epoch": 1.5, "step": 15}
16
+ {"loss": 0.0, "grad_norm": 0.48406901955604553, "learning_rate": 4.849231551964771e-06, "completion_length": 138.4375, "rewards/format_reward_fn": -0.15625, "rewards/env_reward_fn": -1.4037500023841858, "reward": -1.5600000023841858, "reward_std": 0.9590457677841187, "kl": 0.001034960529068485, "epoch": 1.6, "step": 16}
17
+ {"loss": 0.0001, "grad_norm": 0.46559596061706543, "learning_rate": 4.809698831278217e-06, "completion_length": 163.5, "rewards/format_reward_fn": -0.875, "rewards/env_reward_fn": -1.887499988079071, "reward": -2.762500047683716, "reward_std": 0.4749999940395355, "kl": 0.0024155399296432734, "epoch": 1.7, "step": 17}
18
+ {"loss": 0.0, "grad_norm": 0.7438174486160278, "learning_rate": 4.765769467591626e-06, "completion_length": 128.9375, "rewards/format_reward_fn": -0.1875, "rewards/env_reward_fn": -1.2418749928474426, "reward": -1.4293749779462814, "reward_std": 0.7872797846794128, "kl": 0.000904009910300374, "epoch": 1.8, "step": 18}
19
+ {"loss": 0.0001, "grad_norm": 0.01532961055636406, "learning_rate": 4.717527082945555e-06, "completion_length": 225.1875, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0012617496540769935, "epoch": 1.9, "step": 19}
20
+ {"loss": 0.0, "grad_norm": 0.6645981073379517, "learning_rate": 4.665063509461098e-06, "completion_length": 143.5, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.2031250149011612, "reward": -1.296875, "reward_std": 0.40102502703666687, "kl": 0.000525656680110842, "epoch": 2.0, "step": 20}
21
+ {"loss": 0.0, "grad_norm": 0.007929876446723938, "learning_rate": 4.608478614532215e-06, "completion_length": 241.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0007614360365550965, "epoch": 2.1, "step": 21}
22
+ {"loss": 0.0, "grad_norm": 0.38886427879333496, "learning_rate": 4.54788011072248e-06, "completion_length": 133.0625, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.320000022649765, "reward": -1.4137500002980232, "reward_std": 0.2580808103084564, "kl": 0.0011790947755798697, "epoch": 2.2, "step": 22}
23
+ {"loss": 0.0, "grad_norm": 0.5050219297409058, "learning_rate": 4.4833833507280884e-06, "completion_length": 131.875, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.331250011920929, "reward": -1.4250000044703484, "reward_std": 0.25595444440841675, "kl": 0.0009230652649421245, "epoch": 2.3, "step": 23}
24
+ {"loss": 0.0, "grad_norm": 0.01812759041786194, "learning_rate": 4.415111107797445e-06, "completion_length": 200.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0009518663282506168, "epoch": 2.4, "step": 24}
25
+ {"loss": 0.0001, "grad_norm": 0.060016512870788574, "learning_rate": 4.34319334202531e-06, "completion_length": 210.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002712665474973619, "epoch": 2.5, "step": 25}
26
+ {"loss": 0.0001, "grad_norm": 0.05162608623504639, "learning_rate": 4.267766952966369e-06, "completion_length": 182.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002657116623595357, "epoch": 2.6, "step": 26}
27
+ {"loss": 0.0001, "grad_norm": 1.355783462524414, "learning_rate": 4.188975519039151e-06, "completion_length": 194.3125, "rewards/format_reward_fn": -0.90625, "rewards/env_reward_fn": -1.893750011920929, "reward": -2.799999952316284, "reward_std": 0.4000000059604645, "kl": 0.002041113271843642, "epoch": 2.7, "step": 27}
28
+ {"loss": 0.0001, "grad_norm": 0.368874192237854, "learning_rate": 4.106969024216348e-06, "completion_length": 128.75, "rewards/format_reward_fn": -0.125, "rewards/env_reward_fn": -1.5625, "reward": -1.6875, "reward_std": 0.4672893285751343, "kl": 0.0019267259631305933, "epoch": 2.8, "step": 28}
29
+ {"loss": 0.0, "grad_norm": 0.4674609303474426, "learning_rate": 4.021903572521802e-06, "completion_length": 150.125, "rewards/format_reward_fn": -0.09375, "rewards/env_reward_fn": -1.196250006556511, "reward": -1.2899999916553497, "reward_std": 0.45293521881103516, "kl": 0.0008976631215773523, "epoch": 2.9, "step": 29}
30
+ {"loss": 0.0, "grad_norm": 0.5131139755249023, "learning_rate": 3.933941090877615e-06, "completion_length": 150.375, "rewards/format_reward_fn": -0.0625, "rewards/env_reward_fn": -1.1162500083446503, "reward": -1.1787499785423279, "reward_std": 0.3945273756980896, "kl": 0.000526885298313573, "epoch": 3.0, "step": 30}
31
+ {"loss": 0.0001, "grad_norm": 0.37594813108444214, "learning_rate": 3.8432490208670605e-06, "completion_length": 146.375, "rewards/format_reward_fn": -0.25, "rewards/env_reward_fn": -1.550000011920929, "reward": -1.7999999821186066, "reward_std": 0.7966930866241455, "kl": 0.002331543597392738, "epoch": 3.1, "step": 31}
32
+ {"loss": 0.0, "grad_norm": 0.9009740948677063, "learning_rate": 3.7500000000000005e-06, "completion_length": 56.125, "rewards/format_reward_fn": 0.78125, "rewards/env_reward_fn": -0.6912500411272049, "reward": 0.0899999812245369, "reward_std": 0.8214386850595474, "kl": 0.0007165882416302338, "epoch": 3.2, "step": 32}
33
+ {"loss": 0.0, "grad_norm": 0.3561922609806061, "learning_rate": 3.654371533087586e-06, "completion_length": 121.8125, "rewards/format_reward_fn": -0.03125, "rewards/env_reward_fn": -1.3262499868869781, "reward": -1.3574999868869781, "reward_std": 0.4873168468475342, "kl": 0.0006185952515807003, "epoch": 3.3, "step": 33}
34
+ {"loss": 0.0, "grad_norm": 0.001696149935014546, "learning_rate": 3.556545654351749e-06, "completion_length": 256.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0004322115710237995, "epoch": 3.4, "step": 34}
35
+ {"loss": 0.0001, "grad_norm": 0.09834383428096771, "learning_rate": 3.4567085809127247e-06, "completion_length": 196.5, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0030701225623488426, "epoch": 3.5, "step": 35}
36
+ {"loss": 0.0, "grad_norm": 0.5368371605873108, "learning_rate": 3.3550503583141726e-06, "completion_length": 132.4375, "rewards/format_reward_fn": -0.1875, "rewards/env_reward_fn": -1.023477703332901, "reward": -1.2109777331352234, "reward_std": 0.9307937622070312, "kl": 0.0009091942338272929, "epoch": 3.6, "step": 36}
37
+ {"loss": 0.0, "grad_norm": 0.1270669847726822, "learning_rate": 3.2517644987606827e-06, "completion_length": 226.75, "rewards/format_reward_fn": -0.90625, "rewards/env_reward_fn": -1.893750011920929, "reward": -2.799999952316284, "reward_std": 0.4000000059604645, "kl": 0.0010059120832011104, "epoch": 3.7, "step": 37}
38
+ {"loss": 0.0001, "grad_norm": 0.014515245333313942, "learning_rate": 3.147047612756302e-06, "completion_length": 181.125, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0018051152001135051, "epoch": 3.8, "step": 38}
39
+ {"loss": 0.0, "grad_norm": 0.4278377294540405, "learning_rate": 3.0410990348452572e-06, "completion_length": 118.5, "rewards/format_reward_fn": -0.15625, "rewards/env_reward_fn": -1.2950000166893005, "reward": -1.45125000923872, "reward_std": 0.17720277607440948, "kl": 0.0006675019103568047, "epoch": 3.9, "step": 39}
40
+ {"loss": 0.0001, "grad_norm": 1.6945350170135498, "learning_rate": 2.9341204441673267e-06, "completion_length": 37.5625, "rewards/format_reward_fn": 0.5625, "rewards/env_reward_fn": -0.3647203706204891, "reward": 0.1977795884013176, "reward_std": 1.3747702836990356, "kl": 0.0015371840563602746, "epoch": 4.0, "step": 40}
41
+ {"loss": 0.0, "grad_norm": 0.3177756667137146, "learning_rate": 2.82631548055013e-06, "completion_length": 161.1875, "rewards/format_reward_fn": -0.0625, "rewards/env_reward_fn": -1.438749998807907, "reward": -1.501249998807907, "reward_std": 0.5623716115951538, "kl": 0.00039886329614091665, "epoch": 4.1, "step": 41}
42
+ {"loss": 0.0, "grad_norm": 0.0038945211563259363, "learning_rate": 2.717889356869146e-06, "completion_length": 228.75, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0006300912064034492, "epoch": 4.2, "step": 42}
43
+ {"loss": 0.0002, "grad_norm": 1.8007594347000122, "learning_rate": 2.6090484684133406e-06, "completion_length": 199.25, "rewards/format_reward_fn": -0.90625, "rewards/env_reward_fn": -1.893750011920929, "reward": -2.799999952316284, "reward_std": 0.4000000059604645, "kl": 0.0038819790352135897, "epoch": 4.3, "step": 43}
44
+ {"loss": 0.0, "grad_norm": 0.8020444512367249, "learning_rate": 2.5e-06, "completion_length": 51.1875, "rewards/format_reward_fn": 0.78125, "rewards/env_reward_fn": -0.7153906524181366, "reward": 0.0658593624830246, "reward_std": 1.117849439382553, "kl": 0.0010753837414085865, "epoch": 4.4, "step": 44}
45
+ {"loss": 0.0003, "grad_norm": 0.100796177983284, "learning_rate": 2.3909515315866606e-06, "completion_length": 179.125, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.006620461004786193, "epoch": 4.5, "step": 45}
46
+ {"loss": 0.0, "grad_norm": 0.37466055154800415, "learning_rate": 2.2821106431308546e-06, "completion_length": 137.0, "rewards/format_reward_fn": -0.125, "rewards/env_reward_fn": -1.4500000178813934, "reward": -1.5750000029802322, "reward_std": 0.10246553272008896, "kl": 0.0006967572844587266, "epoch": 4.6, "step": 46}
47
+ {"loss": 0.0, "grad_norm": 0.12667995691299438, "learning_rate": 2.173684519449872e-06, "completion_length": 230.625, "rewards/format_reward_fn": -0.90625, "rewards/env_reward_fn": -1.893750011920929, "reward": -2.799999952316284, "reward_std": 0.4000000059604645, "kl": 0.0010832644766196609, "epoch": 4.7, "step": 47}
48
+ {"loss": 0.0002, "grad_norm": 1.7849533557891846, "learning_rate": 2.0658795558326745e-06, "completion_length": 59.9375, "rewards/format_reward_fn": 0.53125, "rewards/env_reward_fn": -0.9174999743700027, "reward": -0.38624997437000275, "reward_std": 1.5010989904403687, "kl": 0.003939345711842179, "epoch": 4.8, "step": 48}
49
+ {"loss": 0.0, "grad_norm": 0.009146175347268581, "learning_rate": 1.958900965154743e-06, "completion_length": 238.4375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0009143970091827214, "epoch": 4.9, "step": 49}
50
+ {"loss": 0.0, "grad_norm": 0.0038632897194474936, "learning_rate": 1.852952387243698e-06, "completion_length": 256.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0005854760529473424, "epoch": 5.0, "step": 50}
51
+ {"loss": 0.0, "grad_norm": 0.0033666163217276335, "learning_rate": 1.7482355012393177e-06, "completion_length": 256.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0006503346667159349, "epoch": 5.1, "step": 51}
52
+ {"loss": 0.0001, "grad_norm": 0.01037515513598919, "learning_rate": 1.6449496416858285e-06, "completion_length": 227.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0015767593868076801, "epoch": 5.2, "step": 52}
53
+ {"loss": 0.0001, "grad_norm": 0.007688457611948252, "learning_rate": 1.5432914190872757e-06, "completion_length": 239.0625, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.001367623801343143, "epoch": 5.3, "step": 53}
54
+ {"loss": 0.0001, "grad_norm": 0.023706091567873955, "learning_rate": 1.443454345648252e-06, "completion_length": 210.5, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0025738890981301665, "epoch": 5.4, "step": 54}
55
+ {"loss": 0.0001, "grad_norm": 0.8433030247688293, "learning_rate": 1.3456284669124159e-06, "completion_length": 130.75, "rewards/format_reward_fn": -0.03125, "rewards/env_reward_fn": -1.3212500214576721, "reward": -1.352500006556511, "reward_std": 0.1381015181541443, "kl": 0.003399069420993328, "epoch": 5.5, "step": 55}
56
+ {"loss": 0.0003, "grad_norm": 0.5677995085716248, "learning_rate": 1.2500000000000007e-06, "completion_length": 138.1875, "rewards/format_reward_fn": -0.03125, "rewards/env_reward_fn": -1.3212500214576721, "reward": -1.3524999916553497, "reward_std": 0.2023404836654663, "kl": 0.0065639110980555415, "epoch": 5.6, "step": 56}
57
+ {"loss": 0.0001, "grad_norm": 0.9626602530479431, "learning_rate": 1.1567509791329402e-06, "completion_length": 197.875, "rewards/format_reward_fn": -0.90625, "rewards/env_reward_fn": -1.881250023841858, "reward": -2.787500023841858, "reward_std": 0.42500001192092896, "kl": 0.002387469750829041, "epoch": 5.7, "step": 57}
58
+ {"loss": 0.0001, "grad_norm": 0.5601871609687805, "learning_rate": 1.0660589091223854e-06, "completion_length": 146.8125, "rewards/format_reward_fn": -0.125, "rewards/env_reward_fn": -1.296875, "reward": -1.421875, "reward_std": 0.2061145156621933, "kl": 0.0014612122904509306, "epoch": 5.8, "step": 58}
59
+ {"loss": 0.0, "grad_norm": 0.0023030152078717947, "learning_rate": 9.780964274781984e-07, "completion_length": 247.8125, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.00044109376904089004, "epoch": 5.9, "step": 59}
60
+ {"loss": 0.0005, "grad_norm": 0.20295704901218414, "learning_rate": 8.930309757836517e-07, "completion_length": 209.3125, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.01255779346683994, "epoch": 6.0, "step": 60}
61
+ {"loss": 0.0002, "grad_norm": 0.08324765413999557, "learning_rate": 8.110244809608494e-07, "completion_length": 214.25, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.005608416337054223, "epoch": 6.1, "step": 61}
62
+ {"loss": 0.0001, "grad_norm": 0.10554463416337967, "learning_rate": 7.322330470336314e-07, "completion_length": 240.4375, "rewards/format_reward_fn": -0.875, "rewards/env_reward_fn": -1.875, "reward": -2.75, "reward_std": 0.5, "kl": 0.003428836236707866, "epoch": 6.2, "step": 62}
63
+ {"loss": 0.0002, "grad_norm": 0.42768168449401855, "learning_rate": 6.568066579746901e-07, "completion_length": 119.0625, "rewards/format_reward_fn": 0.0, "rewards/env_reward_fn": -1.2985156178474426, "reward": -1.298515647649765, "reward_std": 0.35373106598854065, "kl": 0.0057008350268006325, "epoch": 6.3, "step": 63}
64
+ {"loss": 0.0001, "grad_norm": 0.01283896155655384, "learning_rate": 5.848888922025553e-07, "completion_length": 214.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0017039759550243616, "epoch": 6.4, "step": 64}
65
+ {"loss": 0.0001, "grad_norm": 0.6739922165870667, "learning_rate": 5.166166492719124e-07, "completion_length": 146.5, "rewards/format_reward_fn": 0.03125, "rewards/env_reward_fn": -1.0450000166893005, "reward": -1.0137499570846558, "reward_std": 0.7222128808498383, "kl": 0.001302739605307579, "epoch": 6.5, "step": 65}
66
+ {"loss": 0.0003, "grad_norm": 0.17027659714221954, "learning_rate": 4.5211988927752026e-07, "completion_length": 165.75, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.007173331454396248, "epoch": 6.6, "step": 66}
67
+ {"loss": 0.0003, "grad_norm": 0.07421505451202393, "learning_rate": 3.915213854677863e-07, "completion_length": 193.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.00677683437243104, "epoch": 6.7, "step": 67}
68
+ {"loss": 0.0001, "grad_norm": 0.7671226263046265, "learning_rate": 3.3493649053890325e-07, "completion_length": 42.8125, "rewards/format_reward_fn": 0.90625, "rewards/env_reward_fn": -0.3499999940395355, "reward": 0.5562500059604645, "reward_std": 0.7379237115383148, "kl": 0.002484356751665473, "epoch": 6.8, "step": 68}
69
+ {"loss": 0.0001, "grad_norm": 0.05920339748263359, "learning_rate": 2.8247291705444575e-07, "completion_length": 256.0, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0026174949016422033, "epoch": 6.9, "step": 69}
70
+ {"loss": 0.0, "grad_norm": 0.005740954540669918, "learning_rate": 2.3423053240837518e-07, "completion_length": 219.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0009803370048757643, "epoch": 7.0, "step": 70}
71
+ {"loss": 0.0001, "grad_norm": 0.024028871208429337, "learning_rate": 1.9030116872178317e-07, "completion_length": 240.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0021192581625655293, "epoch": 7.1, "step": 71}
72
+ {"loss": 0.0002, "grad_norm": 0.036092352122068405, "learning_rate": 1.507684480352292e-07, "completion_length": 197.1875, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.004276535706594586, "epoch": 7.2, "step": 72}
73
+ {"loss": 0.0001, "grad_norm": 0.02257753349840641, "learning_rate": 1.1570762312943295e-07, "completion_length": 225.625, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002151491877157241, "epoch": 7.3, "step": 73}
74
+ {"loss": 0.0001, "grad_norm": 0.009574878960847855, "learning_rate": 8.518543427732951e-08, "completion_length": 240.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0016366892959922552, "epoch": 7.4, "step": 74}
75
+ {"loss": 0.0001, "grad_norm": 0.028049366548657417, "learning_rate": 5.92599822001666e-08, "completion_length": 224.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002645735861733556, "epoch": 7.5, "step": 75}
76
+ {"loss": 0.0001, "grad_norm": 0.050625696778297424, "learning_rate": 3.798061746947995e-08, "completion_length": 197.625, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0035191430361010134, "epoch": 7.6, "step": 76}
77
+ {"loss": 0.0001, "grad_norm": 0.314410537481308, "learning_rate": 2.1387846565474047e-08, "completion_length": 133.625, "rewards/format_reward_fn": -0.0625, "rewards/env_reward_fn": -1.556249976158142, "reward": -1.6187500059604645, "reward_std": 0.364735871553421, "kl": 0.0014756647869944572, "epoch": 7.7, "step": 77}
78
+ {"loss": 0.0001, "grad_norm": 0.03155818209052086, "learning_rate": 9.513254770636138e-09, "completion_length": 240.5, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.002269520948175341, "epoch": 7.8, "step": 78}
79
+ {"loss": 0.0001, "grad_norm": 0.04115777835249901, "learning_rate": 2.3794460453555046e-09, "completion_length": 240.9375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0034398152492940426, "epoch": 7.9, "step": 79}
80
+ {"loss": 0.0, "grad_norm": 0.0072058564983308315, "learning_rate": 0.0, "completion_length": 241.375, "rewards/format_reward_fn": -1.0, "rewards/env_reward_fn": -2.0, "reward": -3.0, "reward_std": 0.0, "kl": 0.0009698635112727061, "epoch": 8.0, "step": 80}
81
+ {"train_runtime": 2381.7209, "train_samples_per_second": 0.134, "train_steps_per_second": 0.034, "total_flos": 0.0, "train_loss": 8.047953194818546e-05, "epoch": 8.0, "step": 80}
runs/grpo-20260426-062846/per_task_breakdown.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base": {
3
+ "0": {
4
+ "mean": 0.185,
5
+ "std": 0.185,
6
+ "n": 20,
7
+ "positive_pct": 0.5,
8
+ "task": "routine_monday"
9
+ },
10
+ "1": {
11
+ "mean": 0.1792,
12
+ "std": 0.1173139377908695,
13
+ "n": 20,
14
+ "positive_pct": 0.7,
15
+ "task": "storm_surge"
16
+ },
17
+ "2": {
18
+ "mean": 0.1152,
19
+ "std": 0.12735839194964735,
20
+ "n": 20,
21
+ "positive_pct": 0.45,
22
+ "task": "multi_vehicle_pileup"
23
+ },
24
+ "3": {
25
+ "mean": 0.07680000000000001,
26
+ "std": 0.1173139377908695,
27
+ "n": 20,
28
+ "positive_pct": 0.3,
29
+ "task": "fraud_ring_day"
30
+ },
31
+ "4": {
32
+ "mean": 0.1152,
33
+ "std": 0.12735839194964735,
34
+ "n": 20,
35
+ "positive_pct": 0.45,
36
+ "task": "catastrophe_weekend"
37
+ }
38
+ },
39
+ "trained": {
40
+ "0": {
41
+ "mean": 0.185,
42
+ "std": 0.185,
43
+ "n": 20,
44
+ "positive_pct": 0.5,
45
+ "task": "routine_monday"
46
+ },
47
+ "1": {
48
+ "mean": 0.15360000000000001,
49
+ "std": 0.12541387483049873,
50
+ "n": 20,
51
+ "positive_pct": 0.6,
52
+ "task": "storm_surge"
53
+ },
54
+ "2": {
55
+ "mean": 0.21760000000000002,
56
+ "std": 0.09141028388534848,
57
+ "n": 20,
58
+ "positive_pct": 0.85,
59
+ "task": "multi_vehicle_pileup"
60
+ },
61
+ "3": {
62
+ "mean": 0.1024,
63
+ "std": 0.12541387483049873,
64
+ "n": 20,
65
+ "positive_pct": 0.4,
66
+ "task": "fraud_ring_day"
67
+ },
68
+ "4": {
69
+ "mean": 0.1408,
70
+ "std": 0.12735839194964735,
71
+ "n": 20,
72
+ "positive_pct": 0.55,
73
+ "task": "catastrophe_weekend"
74
+ }
75
+ },
76
+ "delta": {
77
+ "0": 0.0,
78
+ "1": -0.025599999999999984,
79
+ "2": 0.10240000000000002,
80
+ "3": 0.025599999999999998,
81
+ "4": 0.02560000000000001
82
+ }
83
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/README.md ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen2.5-1.5B-Instruct
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
7
+ - grpo
8
+ - lora
9
+ - transformers
10
+ - trl
11
+ ---
12
+
13
+ # Model Card for Model ID
14
+
15
+ <!-- Provide a quick summary of what the model is/does. -->
16
+
17
+
18
+
19
+ ## Model Details
20
+
21
+ ### Model Description
22
+
23
+ <!-- Provide a longer summary of what this model is. -->
24
+
25
+
26
+
27
+ - **Developed by:** [More Information Needed]
28
+ - **Funded by [optional]:** [More Information Needed]
29
+ - **Shared by [optional]:** [More Information Needed]
30
+ - **Model type:** [More Information Needed]
31
+ - **Language(s) (NLP):** [More Information Needed]
32
+ - **License:** [More Information Needed]
33
+ - **Finetuned from model [optional]:** [More Information Needed]
34
+
35
+ ### Model Sources [optional]
36
+
37
+ <!-- Provide the basic links for the model. -->
38
+
39
+ - **Repository:** [More Information Needed]
40
+ - **Paper [optional]:** [More Information Needed]
41
+ - **Demo [optional]:** [More Information Needed]
42
+
43
+ ## Uses
44
+
45
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
46
+
47
+ ### Direct Use
48
+
49
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
50
+
51
+ [More Information Needed]
52
+
53
+ ### Downstream Use [optional]
54
+
55
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
56
+
57
+ [More Information Needed]
58
+
59
+ ### Out-of-Scope Use
60
+
61
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
62
+
63
+ [More Information Needed]
64
+
65
+ ## Bias, Risks, and Limitations
66
+
67
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
68
+
69
+ [More Information Needed]
70
+
71
+ ### Recommendations
72
+
73
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
74
+
75
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
76
+
77
+ ## How to Get Started with the Model
78
+
79
+ Use the code below to get started with the model.
80
+
81
+ [More Information Needed]
82
+
83
+ ## Training Details
84
+
85
+ ### Training Data
86
+
87
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
88
+
89
+ [More Information Needed]
90
+
91
+ ### Training Procedure
92
+
93
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
94
+
95
+ #### Preprocessing [optional]
96
+
97
+ [More Information Needed]
98
+
99
+
100
+ #### Training Hyperparameters
101
+
102
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
103
+
104
+ #### Speeds, Sizes, Times [optional]
105
+
106
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
107
+
108
+ [More Information Needed]
109
+
110
+ ## Evaluation
111
+
112
+ <!-- This section describes the evaluation protocols and provides the results. -->
113
+
114
+ ### Testing Data, Factors & Metrics
115
+
116
+ #### Testing Data
117
+
118
+ <!-- This should link to a Dataset Card if possible. -->
119
+
120
+ [More Information Needed]
121
+
122
+ #### Factors
123
+
124
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
125
+
126
+ [More Information Needed]
127
+
128
+ #### Metrics
129
+
130
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
131
+
132
+ [More Information Needed]
133
+
134
+ ### Results
135
+
136
+ [More Information Needed]
137
+
138
+ #### Summary
139
+
140
+
141
+
142
+ ## Model Examination [optional]
143
+
144
+ <!-- Relevant interpretability work for the model goes here -->
145
+
146
+ [More Information Needed]
147
+
148
+ ## Environmental Impact
149
+
150
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
151
+
152
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
153
+
154
+ - **Hardware Type:** [More Information Needed]
155
+ - **Hours used:** [More Information Needed]
156
+ - **Cloud Provider:** [More Information Needed]
157
+ - **Compute Region:** [More Information Needed]
158
+ - **Carbon Emitted:** [More Information Needed]
159
+
160
+ ## Technical Specifications [optional]
161
+
162
+ ### Model Architecture and Objective
163
+
164
+ [More Information Needed]
165
+
166
+ ### Compute Infrastructure
167
+
168
+ [More Information Needed]
169
+
170
+ #### Hardware
171
+
172
+ [More Information Needed]
173
+
174
+ #### Software
175
+
176
+ [More Information Needed]
177
+
178
+ ## Citation [optional]
179
+
180
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
181
+
182
+ **BibTeX:**
183
+
184
+ [More Information Needed]
185
+
186
+ **APA:**
187
+
188
+ [More Information Needed]
189
+
190
+ ## Glossary [optional]
191
+
192
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
193
+
194
+ [More Information Needed]
195
+
196
+ ## More Information [optional]
197
+
198
+ [More Information Needed]
199
+
200
+ ## Model Card Authors [optional]
201
+
202
+ [More Information Needed]
203
+
204
+ ## Model Card Contact
205
+
206
+ [More Information Needed]
207
+ ### Framework versions
208
+
209
+ - PEFT 0.19.1
runs/grpo-20260426-062846/trainer/checkpoint-80/adapter_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 32,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "peft_type": "LORA",
27
+ "peft_version": "0.19.1",
28
+ "qalora_group_size": 16,
29
+ "r": 16,
30
+ "rank_pattern": {},
31
+ "revision": null,
32
+ "target_modules": [
33
+ "down_proj",
34
+ "o_proj",
35
+ "gate_proj",
36
+ "v_proj",
37
+ "k_proj",
38
+ "up_proj",
39
+ "q_proj"
40
+ ],
41
+ "target_parameters": null,
42
+ "task_type": "CAUSAL_LM",
43
+ "trainable_token_indices": null,
44
+ "use_bdlora": null,
45
+ "use_dora": false,
46
+ "use_qalora": false,
47
+ "use_rslora": false
48
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a7c44d68cc79ca09273589fe9c8409651a225d6bdcafe25a1d674df4c1bb199
3
+ size 73911112
runs/grpo-20260426-062846/trainer/checkpoint-80/added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|box_end|>": 151649,
5
+ "<|box_start|>": 151648,
6
+ "<|endoftext|>": 151643,
7
+ "<|file_sep|>": 151664,
8
+ "<|fim_middle|>": 151660,
9
+ "<|fim_pad|>": 151662,
10
+ "<|fim_prefix|>": 151659,
11
+ "<|fim_suffix|>": 151661,
12
+ "<|im_end|>": 151645,
13
+ "<|im_start|>": 151644,
14
+ "<|image_pad|>": 151655,
15
+ "<|object_ref_end|>": 151647,
16
+ "<|object_ref_start|>": 151646,
17
+ "<|quad_end|>": 151651,
18
+ "<|quad_start|>": 151650,
19
+ "<|repo_name|>": 151663,
20
+ "<|video_pad|>": 151656,
21
+ "<|vision_end|>": 151653,
22
+ "<|vision_pad|>": 151654,
23
+ "<|vision_start|>": 151652
24
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
runs/grpo-20260426-062846/trainer/checkpoint-80/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e193009e3d78cdf2a15ebe694062c6e9ff307642ab1335927d471dd4d32402c4
3
+ size 148048187
runs/grpo-20260426-062846/trainer/checkpoint-80/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b32bef304b64dbc27b7144c16801e1879c8ccbf4db13b8a8594a389eb2af29c
3
+ size 14645
runs/grpo-20260426-062846/trainer/checkpoint-80/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cfc6dd1e56d1a7ae20b2507aeaf2806f07a3380121975791614f51e9e718d76
3
+ size 1465
runs/grpo-20260426-062846/trainer/checkpoint-80/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
3
+ size 11422063
runs/grpo-20260426-062846/trainer/checkpoint-80/tokenizer_config.json ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ }
181
+ },
182
+ "additional_special_tokens": [
183
+ "<|im_start|>",
184
+ "<|im_end|>",
185
+ "<|object_ref_start|>",
186
+ "<|object_ref_end|>",
187
+ "<|box_start|>",
188
+ "<|box_end|>",
189
+ "<|quad_start|>",
190
+ "<|quad_end|>",
191
+ "<|vision_start|>",
192
+ "<|vision_end|>",
193
+ "<|vision_pad|>",
194
+ "<|image_pad|>",
195
+ "<|video_pad|>"
196
+ ],
197
+ "bos_token": null,
198
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
199
+ "clean_up_tokenization_spaces": false,
200
+ "eos_token": "<|im_end|>",
201
+ "errors": "replace",
202
+ "extra_special_tokens": {},
203
+ "model_max_length": 131072,
204
+ "pad_token": "<|endoftext|>",
205
+ "split_special_tokens": false,
206
+ "tokenizer_class": "Qwen2Tokenizer",
207
+ "unk_token": null
208
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/trainer_state.json ADDED
@@ -0,0 +1,1073 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 8.0,
5
+ "eval_steps": 500,
6
+ "global_step": 80,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "completion_length": 43.6875,
13
+ "epoch": 0.1,
14
+ "grad_norm": 0.706524133682251,
15
+ "kl": 0.0,
16
+ "learning_rate": 6.25e-07,
17
+ "loss": 0.0,
18
+ "reward": 0.4336448833346367,
19
+ "reward_std": 1.0901877880096436,
20
+ "rewards/env_reward_fn": -0.16010507941246033,
21
+ "rewards/format_reward_fn": 0.59375,
22
+ "step": 1
23
+ },
24
+ {
25
+ "completion_length": 148.6875,
26
+ "epoch": 0.2,
27
+ "grad_norm": 0.46928897500038147,
28
+ "kl": 0.0,
29
+ "learning_rate": 1.25e-06,
30
+ "loss": -0.0,
31
+ "reward": -1.5735156387090683,
32
+ "reward_std": 0.6494516134262085,
33
+ "rewards/env_reward_fn": -1.479765623807907,
34
+ "rewards/format_reward_fn": -0.09375,
35
+ "step": 2
36
+ },
37
+ {
38
+ "completion_length": 241.1875,
39
+ "epoch": 0.3,
40
+ "grad_norm": 0.050346001982688904,
41
+ "kl": 0.0014371701690834016,
42
+ "learning_rate": 1.8750000000000003e-06,
43
+ "loss": 0.0001,
44
+ "reward": -3.0,
45
+ "reward_std": 0.0,
46
+ "rewards/env_reward_fn": -2.0,
47
+ "rewards/format_reward_fn": -1.0,
48
+ "step": 3
49
+ },
50
+ {
51
+ "completion_length": 240.9375,
52
+ "epoch": 0.4,
53
+ "grad_norm": 0.06192959100008011,
54
+ "kl": 0.002273249381687492,
55
+ "learning_rate": 2.5e-06,
56
+ "loss": 0.0001,
57
+ "reward": -3.0,
58
+ "reward_std": 0.0,
59
+ "rewards/env_reward_fn": -2.0,
60
+ "rewards/format_reward_fn": -1.0,
61
+ "step": 4
62
+ },
63
+ {
64
+ "completion_length": 256.0,
65
+ "epoch": 0.5,
66
+ "grad_norm": 0.001998545601963997,
67
+ "kl": 0.0003419504646444693,
68
+ "learning_rate": 3.125e-06,
69
+ "loss": 0.0,
70
+ "reward": -3.0,
71
+ "reward_std": 0.0,
72
+ "rewards/env_reward_fn": -2.0,
73
+ "rewards/format_reward_fn": -1.0,
74
+ "step": 5
75
+ },
76
+ {
77
+ "completion_length": 121.75,
78
+ "epoch": 0.6,
79
+ "grad_norm": 0.3350253701210022,
80
+ "kl": 0.001182895473903045,
81
+ "learning_rate": 3.7500000000000005e-06,
82
+ "loss": 0.0,
83
+ "reward": -1.2899999916553497,
84
+ "reward_std": 0.3629912734031677,
85
+ "rewards/env_reward_fn": -1.1962499916553497,
86
+ "rewards/format_reward_fn": -0.09375,
87
+ "step": 6
88
+ },
89
+ {
90
+ "completion_length": 105.4375,
91
+ "epoch": 0.7,
92
+ "grad_norm": 0.616795539855957,
93
+ "kl": 0.0011062583071179688,
94
+ "learning_rate": 4.3750000000000005e-06,
95
+ "loss": 0.0,
96
+ "reward": -1.3968750089406967,
97
+ "reward_std": 0.2832387089729309,
98
+ "rewards/env_reward_fn": -1.271875023841858,
99
+ "rewards/format_reward_fn": -0.125,
100
+ "step": 7
101
+ },
102
+ {
103
+ "completion_length": 197.875,
104
+ "epoch": 0.8,
105
+ "grad_norm": 0.4317721724510193,
106
+ "kl": 0.0011472782352939248,
107
+ "learning_rate": 5e-06,
108
+ "loss": 0.0,
109
+ "reward": -2.6700000762939453,
110
+ "reward_std": 0.6599999666213989,
111
+ "rewards/env_reward_fn": -1.7950000166893005,
112
+ "rewards/format_reward_fn": -0.875,
113
+ "step": 8
114
+ },
115
+ {
116
+ "completion_length": 149.125,
117
+ "epoch": 0.9,
118
+ "grad_norm": 0.5284587740898132,
119
+ "kl": 0.0006621622887905687,
120
+ "learning_rate": 4.997620553954645e-06,
121
+ "loss": 0.0,
122
+ "reward": -1.3249999564141035,
123
+ "reward_std": 0.5629094392061234,
124
+ "rewards/env_reward_fn": -1.2937500476837158,
125
+ "rewards/format_reward_fn": -0.03125,
126
+ "step": 9
127
+ },
128
+ {
129
+ "completion_length": 158.25,
130
+ "epoch": 1.0,
131
+ "grad_norm": 0.3924754858016968,
132
+ "kl": 0.0005572861700784415,
133
+ "learning_rate": 4.990486745229364e-06,
134
+ "loss": 0.0,
135
+ "reward": -1.2787500023841858,
136
+ "reward_std": 0.2405785173177719,
137
+ "rewards/env_reward_fn": -1.2162500023841858,
138
+ "rewards/format_reward_fn": -0.0625,
139
+ "step": 10
140
+ },
141
+ {
142
+ "completion_length": 226.25,
143
+ "epoch": 1.1,
144
+ "grad_norm": 0.011886635795235634,
145
+ "kl": 0.0009066301281563938,
146
+ "learning_rate": 4.978612153434527e-06,
147
+ "loss": 0.0,
148
+ "reward": -3.0,
149
+ "reward_std": 0.0,
150
+ "rewards/env_reward_fn": -2.0,
151
+ "rewards/format_reward_fn": -1.0,
152
+ "step": 11
153
+ },
154
+ {
155
+ "completion_length": 229.4375,
156
+ "epoch": 1.2,
157
+ "grad_norm": 0.00472637265920639,
158
+ "kl": 0.0004253629158483818,
159
+ "learning_rate": 4.962019382530521e-06,
160
+ "loss": 0.0,
161
+ "reward": -3.0,
162
+ "reward_std": 0.0,
163
+ "rewards/env_reward_fn": -2.0,
164
+ "rewards/format_reward_fn": -1.0,
165
+ "step": 12
166
+ },
167
+ {
168
+ "completion_length": 132.75,
169
+ "epoch": 1.3,
170
+ "grad_norm": 0.623696506023407,
171
+ "kl": 0.0006600431515835226,
172
+ "learning_rate": 4.9407400177998335e-06,
173
+ "loss": 0.0,
174
+ "reward": -1.4374999403953552,
175
+ "reward_std": 0.8298223614692688,
176
+ "rewards/env_reward_fn": -1.4375000596046448,
177
+ "rewards/format_reward_fn": 0.0,
178
+ "step": 13
179
+ },
180
+ {
181
+ "completion_length": 134.375,
182
+ "epoch": 1.4,
183
+ "grad_norm": 0.398609459400177,
184
+ "kl": 0.000562123314011842,
185
+ "learning_rate": 4.914814565722671e-06,
186
+ "loss": 0.0,
187
+ "reward": -1.5824999958276749,
188
+ "reward_std": 0.536240816116333,
189
+ "rewards/env_reward_fn": -1.3949999809265137,
190
+ "rewards/format_reward_fn": -0.1875,
191
+ "step": 14
192
+ },
193
+ {
194
+ "completion_length": 240.5,
195
+ "epoch": 1.5,
196
+ "grad_norm": 0.009604142978787422,
197
+ "kl": 0.0008011562749743462,
198
+ "learning_rate": 4.884292376870567e-06,
199
+ "loss": 0.0,
200
+ "reward": -3.0,
201
+ "reward_std": 0.0,
202
+ "rewards/env_reward_fn": -2.0,
203
+ "rewards/format_reward_fn": -1.0,
204
+ "step": 15
205
+ },
206
+ {
207
+ "completion_length": 138.4375,
208
+ "epoch": 1.6,
209
+ "grad_norm": 0.48406901955604553,
210
+ "kl": 0.001034960529068485,
211
+ "learning_rate": 4.849231551964771e-06,
212
+ "loss": 0.0,
213
+ "reward": -1.5600000023841858,
214
+ "reward_std": 0.9590457677841187,
215
+ "rewards/env_reward_fn": -1.4037500023841858,
216
+ "rewards/format_reward_fn": -0.15625,
217
+ "step": 16
218
+ },
219
+ {
220
+ "completion_length": 163.5,
221
+ "epoch": 1.7,
222
+ "grad_norm": 0.46559596061706543,
223
+ "kl": 0.0024155399296432734,
224
+ "learning_rate": 4.809698831278217e-06,
225
+ "loss": 0.0001,
226
+ "reward": -2.762500047683716,
227
+ "reward_std": 0.4749999940395355,
228
+ "rewards/env_reward_fn": -1.887499988079071,
229
+ "rewards/format_reward_fn": -0.875,
230
+ "step": 17
231
+ },
232
+ {
233
+ "completion_length": 128.9375,
234
+ "epoch": 1.8,
235
+ "grad_norm": 0.7438174486160278,
236
+ "kl": 0.000904009910300374,
237
+ "learning_rate": 4.765769467591626e-06,
238
+ "loss": 0.0,
239
+ "reward": -1.4293749779462814,
240
+ "reward_std": 0.7872797846794128,
241
+ "rewards/env_reward_fn": -1.2418749928474426,
242
+ "rewards/format_reward_fn": -0.1875,
243
+ "step": 18
244
+ },
245
+ {
246
+ "completion_length": 225.1875,
247
+ "epoch": 1.9,
248
+ "grad_norm": 0.01532961055636406,
249
+ "kl": 0.0012617496540769935,
250
+ "learning_rate": 4.717527082945555e-06,
251
+ "loss": 0.0001,
252
+ "reward": -3.0,
253
+ "reward_std": 0.0,
254
+ "rewards/env_reward_fn": -2.0,
255
+ "rewards/format_reward_fn": -1.0,
256
+ "step": 19
257
+ },
258
+ {
259
+ "completion_length": 143.5,
260
+ "epoch": 2.0,
261
+ "grad_norm": 0.6645981073379517,
262
+ "kl": 0.000525656680110842,
263
+ "learning_rate": 4.665063509461098e-06,
264
+ "loss": 0.0,
265
+ "reward": -1.296875,
266
+ "reward_std": 0.40102502703666687,
267
+ "rewards/env_reward_fn": -1.2031250149011612,
268
+ "rewards/format_reward_fn": -0.09375,
269
+ "step": 20
270
+ },
271
+ {
272
+ "completion_length": 241.375,
273
+ "epoch": 2.1,
274
+ "grad_norm": 0.007929876446723938,
275
+ "kl": 0.0007614360365550965,
276
+ "learning_rate": 4.608478614532215e-06,
277
+ "loss": 0.0,
278
+ "reward": -3.0,
279
+ "reward_std": 0.0,
280
+ "rewards/env_reward_fn": -2.0,
281
+ "rewards/format_reward_fn": -1.0,
282
+ "step": 21
283
+ },
284
+ {
285
+ "completion_length": 133.0625,
286
+ "epoch": 2.2,
287
+ "grad_norm": 0.38886427879333496,
288
+ "kl": 0.0011790947755798697,
289
+ "learning_rate": 4.54788011072248e-06,
290
+ "loss": 0.0,
291
+ "reward": -1.4137500002980232,
292
+ "reward_std": 0.2580808103084564,
293
+ "rewards/env_reward_fn": -1.320000022649765,
294
+ "rewards/format_reward_fn": -0.09375,
295
+ "step": 22
296
+ },
297
+ {
298
+ "completion_length": 131.875,
299
+ "epoch": 2.3,
300
+ "grad_norm": 0.5050219297409058,
301
+ "kl": 0.0009230652649421245,
302
+ "learning_rate": 4.4833833507280884e-06,
303
+ "loss": 0.0,
304
+ "reward": -1.4250000044703484,
305
+ "reward_std": 0.25595444440841675,
306
+ "rewards/env_reward_fn": -1.331250011920929,
307
+ "rewards/format_reward_fn": -0.09375,
308
+ "step": 23
309
+ },
310
+ {
311
+ "completion_length": 200.375,
312
+ "epoch": 2.4,
313
+ "grad_norm": 0.01812759041786194,
314
+ "kl": 0.0009518663282506168,
315
+ "learning_rate": 4.415111107797445e-06,
316
+ "loss": 0.0,
317
+ "reward": -3.0,
318
+ "reward_std": 0.0,
319
+ "rewards/env_reward_fn": -2.0,
320
+ "rewards/format_reward_fn": -1.0,
321
+ "step": 24
322
+ },
323
+ {
324
+ "completion_length": 210.9375,
325
+ "epoch": 2.5,
326
+ "grad_norm": 0.060016512870788574,
327
+ "kl": 0.002712665474973619,
328
+ "learning_rate": 4.34319334202531e-06,
329
+ "loss": 0.0001,
330
+ "reward": -3.0,
331
+ "reward_std": 0.0,
332
+ "rewards/env_reward_fn": -2.0,
333
+ "rewards/format_reward_fn": -1.0,
334
+ "step": 25
335
+ },
336
+ {
337
+ "completion_length": 182.375,
338
+ "epoch": 2.6,
339
+ "grad_norm": 0.05162608623504639,
340
+ "kl": 0.002657116623595357,
341
+ "learning_rate": 4.267766952966369e-06,
342
+ "loss": 0.0001,
343
+ "reward": -3.0,
344
+ "reward_std": 0.0,
345
+ "rewards/env_reward_fn": -2.0,
346
+ "rewards/format_reward_fn": -1.0,
347
+ "step": 26
348
+ },
349
+ {
350
+ "completion_length": 194.3125,
351
+ "epoch": 2.7,
352
+ "grad_norm": 1.355783462524414,
353
+ "kl": 0.002041113271843642,
354
+ "learning_rate": 4.188975519039151e-06,
355
+ "loss": 0.0001,
356
+ "reward": -2.799999952316284,
357
+ "reward_std": 0.4000000059604645,
358
+ "rewards/env_reward_fn": -1.893750011920929,
359
+ "rewards/format_reward_fn": -0.90625,
360
+ "step": 27
361
+ },
362
+ {
363
+ "completion_length": 128.75,
364
+ "epoch": 2.8,
365
+ "grad_norm": 0.368874192237854,
366
+ "kl": 0.0019267259631305933,
367
+ "learning_rate": 4.106969024216348e-06,
368
+ "loss": 0.0001,
369
+ "reward": -1.6875,
370
+ "reward_std": 0.4672893285751343,
371
+ "rewards/env_reward_fn": -1.5625,
372
+ "rewards/format_reward_fn": -0.125,
373
+ "step": 28
374
+ },
375
+ {
376
+ "completion_length": 150.125,
377
+ "epoch": 2.9,
378
+ "grad_norm": 0.4674609303474426,
379
+ "kl": 0.0008976631215773523,
380
+ "learning_rate": 4.021903572521802e-06,
381
+ "loss": 0.0,
382
+ "reward": -1.2899999916553497,
383
+ "reward_std": 0.45293521881103516,
384
+ "rewards/env_reward_fn": -1.196250006556511,
385
+ "rewards/format_reward_fn": -0.09375,
386
+ "step": 29
387
+ },
388
+ {
389
+ "completion_length": 150.375,
390
+ "epoch": 3.0,
391
+ "grad_norm": 0.5131139755249023,
392
+ "kl": 0.000526885298313573,
393
+ "learning_rate": 3.933941090877615e-06,
394
+ "loss": 0.0,
395
+ "reward": -1.1787499785423279,
396
+ "reward_std": 0.3945273756980896,
397
+ "rewards/env_reward_fn": -1.1162500083446503,
398
+ "rewards/format_reward_fn": -0.0625,
399
+ "step": 30
400
+ },
401
+ {
402
+ "completion_length": 146.375,
403
+ "epoch": 3.1,
404
+ "grad_norm": 0.37594813108444214,
405
+ "kl": 0.002331543597392738,
406
+ "learning_rate": 3.8432490208670605e-06,
407
+ "loss": 0.0001,
408
+ "reward": -1.7999999821186066,
409
+ "reward_std": 0.7966930866241455,
410
+ "rewards/env_reward_fn": -1.550000011920929,
411
+ "rewards/format_reward_fn": -0.25,
412
+ "step": 31
413
+ },
414
+ {
415
+ "completion_length": 56.125,
416
+ "epoch": 3.2,
417
+ "grad_norm": 0.9009740948677063,
418
+ "kl": 0.0007165882416302338,
419
+ "learning_rate": 3.7500000000000005e-06,
420
+ "loss": 0.0,
421
+ "reward": 0.0899999812245369,
422
+ "reward_std": 0.8214386850595474,
423
+ "rewards/env_reward_fn": -0.6912500411272049,
424
+ "rewards/format_reward_fn": 0.78125,
425
+ "step": 32
426
+ },
427
+ {
428
+ "completion_length": 121.8125,
429
+ "epoch": 3.3,
430
+ "grad_norm": 0.3561922609806061,
431
+ "kl": 0.0006185952515807003,
432
+ "learning_rate": 3.654371533087586e-06,
433
+ "loss": 0.0,
434
+ "reward": -1.3574999868869781,
435
+ "reward_std": 0.4873168468475342,
436
+ "rewards/env_reward_fn": -1.3262499868869781,
437
+ "rewards/format_reward_fn": -0.03125,
438
+ "step": 33
439
+ },
440
+ {
441
+ "completion_length": 256.0,
442
+ "epoch": 3.4,
443
+ "grad_norm": 0.001696149935014546,
444
+ "kl": 0.0004322115710237995,
445
+ "learning_rate": 3.556545654351749e-06,
446
+ "loss": 0.0,
447
+ "reward": -3.0,
448
+ "reward_std": 0.0,
449
+ "rewards/env_reward_fn": -2.0,
450
+ "rewards/format_reward_fn": -1.0,
451
+ "step": 34
452
+ },
453
+ {
454
+ "completion_length": 196.5,
455
+ "epoch": 3.5,
456
+ "grad_norm": 0.09834383428096771,
457
+ "kl": 0.0030701225623488426,
458
+ "learning_rate": 3.4567085809127247e-06,
459
+ "loss": 0.0001,
460
+ "reward": -3.0,
461
+ "reward_std": 0.0,
462
+ "rewards/env_reward_fn": -2.0,
463
+ "rewards/format_reward_fn": -1.0,
464
+ "step": 35
465
+ },
466
+ {
467
+ "completion_length": 132.4375,
468
+ "epoch": 3.6,
469
+ "grad_norm": 0.5368371605873108,
470
+ "kl": 0.0009091942338272929,
471
+ "learning_rate": 3.3550503583141726e-06,
472
+ "loss": 0.0,
473
+ "reward": -1.2109777331352234,
474
+ "reward_std": 0.9307937622070312,
475
+ "rewards/env_reward_fn": -1.023477703332901,
476
+ "rewards/format_reward_fn": -0.1875,
477
+ "step": 36
478
+ },
479
+ {
480
+ "completion_length": 226.75,
481
+ "epoch": 3.7,
482
+ "grad_norm": 0.1270669847726822,
483
+ "kl": 0.0010059120832011104,
484
+ "learning_rate": 3.2517644987606827e-06,
485
+ "loss": 0.0,
486
+ "reward": -2.799999952316284,
487
+ "reward_std": 0.4000000059604645,
488
+ "rewards/env_reward_fn": -1.893750011920929,
489
+ "rewards/format_reward_fn": -0.90625,
490
+ "step": 37
491
+ },
492
+ {
493
+ "completion_length": 181.125,
494
+ "epoch": 3.8,
495
+ "grad_norm": 0.014515245333313942,
496
+ "kl": 0.0018051152001135051,
497
+ "learning_rate": 3.147047612756302e-06,
498
+ "loss": 0.0001,
499
+ "reward": -3.0,
500
+ "reward_std": 0.0,
501
+ "rewards/env_reward_fn": -2.0,
502
+ "rewards/format_reward_fn": -1.0,
503
+ "step": 38
504
+ },
505
+ {
506
+ "completion_length": 118.5,
507
+ "epoch": 3.9,
508
+ "grad_norm": 0.4278377294540405,
509
+ "kl": 0.0006675019103568047,
510
+ "learning_rate": 3.0410990348452572e-06,
511
+ "loss": 0.0,
512
+ "reward": -1.45125000923872,
513
+ "reward_std": 0.17720277607440948,
514
+ "rewards/env_reward_fn": -1.2950000166893005,
515
+ "rewards/format_reward_fn": -0.15625,
516
+ "step": 39
517
+ },
518
+ {
519
+ "completion_length": 37.5625,
520
+ "epoch": 4.0,
521
+ "grad_norm": 1.6945350170135498,
522
+ "kl": 0.0015371840563602746,
523
+ "learning_rate": 2.9341204441673267e-06,
524
+ "loss": 0.0001,
525
+ "reward": 0.1977795884013176,
526
+ "reward_std": 1.3747702836990356,
527
+ "rewards/env_reward_fn": -0.3647203706204891,
528
+ "rewards/format_reward_fn": 0.5625,
529
+ "step": 40
530
+ },
531
+ {
532
+ "completion_length": 161.1875,
533
+ "epoch": 4.1,
534
+ "grad_norm": 0.3177756667137146,
535
+ "kl": 0.00039886329614091665,
536
+ "learning_rate": 2.82631548055013e-06,
537
+ "loss": 0.0,
538
+ "reward": -1.501249998807907,
539
+ "reward_std": 0.5623716115951538,
540
+ "rewards/env_reward_fn": -1.438749998807907,
541
+ "rewards/format_reward_fn": -0.0625,
542
+ "step": 41
543
+ },
544
+ {
545
+ "completion_length": 228.75,
546
+ "epoch": 4.2,
547
+ "grad_norm": 0.0038945211563259363,
548
+ "kl": 0.0006300912064034492,
549
+ "learning_rate": 2.717889356869146e-06,
550
+ "loss": 0.0,
551
+ "reward": -3.0,
552
+ "reward_std": 0.0,
553
+ "rewards/env_reward_fn": -2.0,
554
+ "rewards/format_reward_fn": -1.0,
555
+ "step": 42
556
+ },
557
+ {
558
+ "completion_length": 199.25,
559
+ "epoch": 4.3,
560
+ "grad_norm": 1.8007594347000122,
561
+ "kl": 0.0038819790352135897,
562
+ "learning_rate": 2.6090484684133406e-06,
563
+ "loss": 0.0002,
564
+ "reward": -2.799999952316284,
565
+ "reward_std": 0.4000000059604645,
566
+ "rewards/env_reward_fn": -1.893750011920929,
567
+ "rewards/format_reward_fn": -0.90625,
568
+ "step": 43
569
+ },
570
+ {
571
+ "completion_length": 51.1875,
572
+ "epoch": 4.4,
573
+ "grad_norm": 0.8020444512367249,
574
+ "kl": 0.0010753837414085865,
575
+ "learning_rate": 2.5e-06,
576
+ "loss": 0.0,
577
+ "reward": 0.0658593624830246,
578
+ "reward_std": 1.117849439382553,
579
+ "rewards/env_reward_fn": -0.7153906524181366,
580
+ "rewards/format_reward_fn": 0.78125,
581
+ "step": 44
582
+ },
583
+ {
584
+ "completion_length": 179.125,
585
+ "epoch": 4.5,
586
+ "grad_norm": 0.100796177983284,
587
+ "kl": 0.006620461004786193,
588
+ "learning_rate": 2.3909515315866606e-06,
589
+ "loss": 0.0003,
590
+ "reward": -3.0,
591
+ "reward_std": 0.0,
592
+ "rewards/env_reward_fn": -2.0,
593
+ "rewards/format_reward_fn": -1.0,
594
+ "step": 45
595
+ },
596
+ {
597
+ "completion_length": 137.0,
598
+ "epoch": 4.6,
599
+ "grad_norm": 0.37466055154800415,
600
+ "kl": 0.0006967572844587266,
601
+ "learning_rate": 2.2821106431308546e-06,
602
+ "loss": 0.0,
603
+ "reward": -1.5750000029802322,
604
+ "reward_std": 0.10246553272008896,
605
+ "rewards/env_reward_fn": -1.4500000178813934,
606
+ "rewards/format_reward_fn": -0.125,
607
+ "step": 46
608
+ },
609
+ {
610
+ "completion_length": 230.625,
611
+ "epoch": 4.7,
612
+ "grad_norm": 0.12667995691299438,
613
+ "kl": 0.0010832644766196609,
614
+ "learning_rate": 2.173684519449872e-06,
615
+ "loss": 0.0,
616
+ "reward": -2.799999952316284,
617
+ "reward_std": 0.4000000059604645,
618
+ "rewards/env_reward_fn": -1.893750011920929,
619
+ "rewards/format_reward_fn": -0.90625,
620
+ "step": 47
621
+ },
622
+ {
623
+ "completion_length": 59.9375,
624
+ "epoch": 4.8,
625
+ "grad_norm": 1.7849533557891846,
626
+ "kl": 0.003939345711842179,
627
+ "learning_rate": 2.0658795558326745e-06,
628
+ "loss": 0.0002,
629
+ "reward": -0.38624997437000275,
630
+ "reward_std": 1.5010989904403687,
631
+ "rewards/env_reward_fn": -0.9174999743700027,
632
+ "rewards/format_reward_fn": 0.53125,
633
+ "step": 48
634
+ },
635
+ {
636
+ "completion_length": 238.4375,
637
+ "epoch": 4.9,
638
+ "grad_norm": 0.009146175347268581,
639
+ "kl": 0.0009143970091827214,
640
+ "learning_rate": 1.958900965154743e-06,
641
+ "loss": 0.0,
642
+ "reward": -3.0,
643
+ "reward_std": 0.0,
644
+ "rewards/env_reward_fn": -2.0,
645
+ "rewards/format_reward_fn": -1.0,
646
+ "step": 49
647
+ },
648
+ {
649
+ "completion_length": 256.0,
650
+ "epoch": 5.0,
651
+ "grad_norm": 0.0038632897194474936,
652
+ "kl": 0.0005854760529473424,
653
+ "learning_rate": 1.852952387243698e-06,
654
+ "loss": 0.0,
655
+ "reward": -3.0,
656
+ "reward_std": 0.0,
657
+ "rewards/env_reward_fn": -2.0,
658
+ "rewards/format_reward_fn": -1.0,
659
+ "step": 50
660
+ },
661
+ {
662
+ "completion_length": 256.0,
663
+ "epoch": 5.1,
664
+ "grad_norm": 0.0033666163217276335,
665
+ "kl": 0.0006503346667159349,
666
+ "learning_rate": 1.7482355012393177e-06,
667
+ "loss": 0.0,
668
+ "reward": -3.0,
669
+ "reward_std": 0.0,
670
+ "rewards/env_reward_fn": -2.0,
671
+ "rewards/format_reward_fn": -1.0,
672
+ "step": 51
673
+ },
674
+ {
675
+ "completion_length": 227.0,
676
+ "epoch": 5.2,
677
+ "grad_norm": 0.01037515513598919,
678
+ "kl": 0.0015767593868076801,
679
+ "learning_rate": 1.6449496416858285e-06,
680
+ "loss": 0.0001,
681
+ "reward": -3.0,
682
+ "reward_std": 0.0,
683
+ "rewards/env_reward_fn": -2.0,
684
+ "rewards/format_reward_fn": -1.0,
685
+ "step": 52
686
+ },
687
+ {
688
+ "completion_length": 239.0625,
689
+ "epoch": 5.3,
690
+ "grad_norm": 0.007688457611948252,
691
+ "kl": 0.001367623801343143,
692
+ "learning_rate": 1.5432914190872757e-06,
693
+ "loss": 0.0001,
694
+ "reward": -3.0,
695
+ "reward_std": 0.0,
696
+ "rewards/env_reward_fn": -2.0,
697
+ "rewards/format_reward_fn": -1.0,
698
+ "step": 53
699
+ },
700
+ {
701
+ "completion_length": 210.5,
702
+ "epoch": 5.4,
703
+ "grad_norm": 0.023706091567873955,
704
+ "kl": 0.0025738890981301665,
705
+ "learning_rate": 1.443454345648252e-06,
706
+ "loss": 0.0001,
707
+ "reward": -3.0,
708
+ "reward_std": 0.0,
709
+ "rewards/env_reward_fn": -2.0,
710
+ "rewards/format_reward_fn": -1.0,
711
+ "step": 54
712
+ },
713
+ {
714
+ "completion_length": 130.75,
715
+ "epoch": 5.5,
716
+ "grad_norm": 0.8433030247688293,
717
+ "kl": 0.003399069420993328,
718
+ "learning_rate": 1.3456284669124159e-06,
719
+ "loss": 0.0001,
720
+ "reward": -1.352500006556511,
721
+ "reward_std": 0.1381015181541443,
722
+ "rewards/env_reward_fn": -1.3212500214576721,
723
+ "rewards/format_reward_fn": -0.03125,
724
+ "step": 55
725
+ },
726
+ {
727
+ "completion_length": 138.1875,
728
+ "epoch": 5.6,
729
+ "grad_norm": 0.5677995085716248,
730
+ "kl": 0.0065639110980555415,
731
+ "learning_rate": 1.2500000000000007e-06,
732
+ "loss": 0.0003,
733
+ "reward": -1.3524999916553497,
734
+ "reward_std": 0.2023404836654663,
735
+ "rewards/env_reward_fn": -1.3212500214576721,
736
+ "rewards/format_reward_fn": -0.03125,
737
+ "step": 56
738
+ },
739
+ {
740
+ "completion_length": 197.875,
741
+ "epoch": 5.7,
742
+ "grad_norm": 0.9626602530479431,
743
+ "kl": 0.002387469750829041,
744
+ "learning_rate": 1.1567509791329402e-06,
745
+ "loss": 0.0001,
746
+ "reward": -2.787500023841858,
747
+ "reward_std": 0.42500001192092896,
748
+ "rewards/env_reward_fn": -1.881250023841858,
749
+ "rewards/format_reward_fn": -0.90625,
750
+ "step": 57
751
+ },
752
+ {
753
+ "completion_length": 146.8125,
754
+ "epoch": 5.8,
755
+ "grad_norm": 0.5601871609687805,
756
+ "kl": 0.0014612122904509306,
757
+ "learning_rate": 1.0660589091223854e-06,
758
+ "loss": 0.0001,
759
+ "reward": -1.421875,
760
+ "reward_std": 0.2061145156621933,
761
+ "rewards/env_reward_fn": -1.296875,
762
+ "rewards/format_reward_fn": -0.125,
763
+ "step": 58
764
+ },
765
+ {
766
+ "completion_length": 247.8125,
767
+ "epoch": 5.9,
768
+ "grad_norm": 0.0023030152078717947,
769
+ "kl": 0.00044109376904089004,
770
+ "learning_rate": 9.780964274781984e-07,
771
+ "loss": 0.0,
772
+ "reward": -3.0,
773
+ "reward_std": 0.0,
774
+ "rewards/env_reward_fn": -2.0,
775
+ "rewards/format_reward_fn": -1.0,
776
+ "step": 59
777
+ },
778
+ {
779
+ "completion_length": 209.3125,
780
+ "epoch": 6.0,
781
+ "grad_norm": 0.20295704901218414,
782
+ "kl": 0.01255779346683994,
783
+ "learning_rate": 8.930309757836517e-07,
784
+ "loss": 0.0005,
785
+ "reward": -3.0,
786
+ "reward_std": 0.0,
787
+ "rewards/env_reward_fn": -2.0,
788
+ "rewards/format_reward_fn": -1.0,
789
+ "step": 60
790
+ },
791
+ {
792
+ "completion_length": 214.25,
793
+ "epoch": 6.1,
794
+ "grad_norm": 0.08324765413999557,
795
+ "kl": 0.005608416337054223,
796
+ "learning_rate": 8.110244809608494e-07,
797
+ "loss": 0.0002,
798
+ "reward": -3.0,
799
+ "reward_std": 0.0,
800
+ "rewards/env_reward_fn": -2.0,
801
+ "rewards/format_reward_fn": -1.0,
802
+ "step": 61
803
+ },
804
+ {
805
+ "completion_length": 240.4375,
806
+ "epoch": 6.2,
807
+ "grad_norm": 0.10554463416337967,
808
+ "kl": 0.003428836236707866,
809
+ "learning_rate": 7.322330470336314e-07,
810
+ "loss": 0.0001,
811
+ "reward": -2.75,
812
+ "reward_std": 0.5,
813
+ "rewards/env_reward_fn": -1.875,
814
+ "rewards/format_reward_fn": -0.875,
815
+ "step": 62
816
+ },
817
+ {
818
+ "completion_length": 119.0625,
819
+ "epoch": 6.3,
820
+ "grad_norm": 0.42768168449401855,
821
+ "kl": 0.0057008350268006325,
822
+ "learning_rate": 6.568066579746901e-07,
823
+ "loss": 0.0002,
824
+ "reward": -1.298515647649765,
825
+ "reward_std": 0.35373106598854065,
826
+ "rewards/env_reward_fn": -1.2985156178474426,
827
+ "rewards/format_reward_fn": 0.0,
828
+ "step": 63
829
+ },
830
+ {
831
+ "completion_length": 214.375,
832
+ "epoch": 6.4,
833
+ "grad_norm": 0.01283896155655384,
834
+ "kl": 0.0017039759550243616,
835
+ "learning_rate": 5.848888922025553e-07,
836
+ "loss": 0.0001,
837
+ "reward": -3.0,
838
+ "reward_std": 0.0,
839
+ "rewards/env_reward_fn": -2.0,
840
+ "rewards/format_reward_fn": -1.0,
841
+ "step": 64
842
+ },
843
+ {
844
+ "completion_length": 146.5,
845
+ "epoch": 6.5,
846
+ "grad_norm": 0.6739922165870667,
847
+ "kl": 0.001302739605307579,
848
+ "learning_rate": 5.166166492719124e-07,
849
+ "loss": 0.0001,
850
+ "reward": -1.0137499570846558,
851
+ "reward_std": 0.7222128808498383,
852
+ "rewards/env_reward_fn": -1.0450000166893005,
853
+ "rewards/format_reward_fn": 0.03125,
854
+ "step": 65
855
+ },
856
+ {
857
+ "completion_length": 165.75,
858
+ "epoch": 6.6,
859
+ "grad_norm": 0.17027659714221954,
860
+ "kl": 0.007173331454396248,
861
+ "learning_rate": 4.5211988927752026e-07,
862
+ "loss": 0.0003,
863
+ "reward": -3.0,
864
+ "reward_std": 0.0,
865
+ "rewards/env_reward_fn": -2.0,
866
+ "rewards/format_reward_fn": -1.0,
867
+ "step": 66
868
+ },
869
+ {
870
+ "completion_length": 193.9375,
871
+ "epoch": 6.7,
872
+ "grad_norm": 0.07421505451202393,
873
+ "kl": 0.00677683437243104,
874
+ "learning_rate": 3.915213854677863e-07,
875
+ "loss": 0.0003,
876
+ "reward": -3.0,
877
+ "reward_std": 0.0,
878
+ "rewards/env_reward_fn": -2.0,
879
+ "rewards/format_reward_fn": -1.0,
880
+ "step": 67
881
+ },
882
+ {
883
+ "completion_length": 42.8125,
884
+ "epoch": 6.8,
885
+ "grad_norm": 0.7671226263046265,
886
+ "kl": 0.002484356751665473,
887
+ "learning_rate": 3.3493649053890325e-07,
888
+ "loss": 0.0001,
889
+ "reward": 0.5562500059604645,
890
+ "reward_std": 0.7379237115383148,
891
+ "rewards/env_reward_fn": -0.3499999940395355,
892
+ "rewards/format_reward_fn": 0.90625,
893
+ "step": 68
894
+ },
895
+ {
896
+ "completion_length": 256.0,
897
+ "epoch": 6.9,
898
+ "grad_norm": 0.05920339748263359,
899
+ "kl": 0.0026174949016422033,
900
+ "learning_rate": 2.8247291705444575e-07,
901
+ "loss": 0.0001,
902
+ "reward": -3.0,
903
+ "reward_std": 0.0,
904
+ "rewards/env_reward_fn": -2.0,
905
+ "rewards/format_reward_fn": -1.0,
906
+ "step": 69
907
+ },
908
+ {
909
+ "completion_length": 219.9375,
910
+ "epoch": 7.0,
911
+ "grad_norm": 0.005740954540669918,
912
+ "kl": 0.0009803370048757643,
913
+ "learning_rate": 2.3423053240837518e-07,
914
+ "loss": 0.0,
915
+ "reward": -3.0,
916
+ "reward_std": 0.0,
917
+ "rewards/env_reward_fn": -2.0,
918
+ "rewards/format_reward_fn": -1.0,
919
+ "step": 70
920
+ },
921
+ {
922
+ "completion_length": 240.375,
923
+ "epoch": 7.1,
924
+ "grad_norm": 0.024028871208429337,
925
+ "kl": 0.0021192581625655293,
926
+ "learning_rate": 1.9030116872178317e-07,
927
+ "loss": 0.0001,
928
+ "reward": -3.0,
929
+ "reward_std": 0.0,
930
+ "rewards/env_reward_fn": -2.0,
931
+ "rewards/format_reward_fn": -1.0,
932
+ "step": 71
933
+ },
934
+ {
935
+ "completion_length": 197.1875,
936
+ "epoch": 7.2,
937
+ "grad_norm": 0.036092352122068405,
938
+ "kl": 0.004276535706594586,
939
+ "learning_rate": 1.507684480352292e-07,
940
+ "loss": 0.0002,
941
+ "reward": -3.0,
942
+ "reward_std": 0.0,
943
+ "rewards/env_reward_fn": -2.0,
944
+ "rewards/format_reward_fn": -1.0,
945
+ "step": 72
946
+ },
947
+ {
948
+ "completion_length": 225.625,
949
+ "epoch": 7.3,
950
+ "grad_norm": 0.02257753349840641,
951
+ "kl": 0.002151491877157241,
952
+ "learning_rate": 1.1570762312943295e-07,
953
+ "loss": 0.0001,
954
+ "reward": -3.0,
955
+ "reward_std": 0.0,
956
+ "rewards/env_reward_fn": -2.0,
957
+ "rewards/format_reward_fn": -1.0,
958
+ "step": 73
959
+ },
960
+ {
961
+ "completion_length": 240.9375,
962
+ "epoch": 7.4,
963
+ "grad_norm": 0.009574878960847855,
964
+ "kl": 0.0016366892959922552,
965
+ "learning_rate": 8.518543427732951e-08,
966
+ "loss": 0.0001,
967
+ "reward": -3.0,
968
+ "reward_std": 0.0,
969
+ "rewards/env_reward_fn": -2.0,
970
+ "rewards/format_reward_fn": -1.0,
971
+ "step": 74
972
+ },
973
+ {
974
+ "completion_length": 224.9375,
975
+ "epoch": 7.5,
976
+ "grad_norm": 0.028049366548657417,
977
+ "kl": 0.002645735861733556,
978
+ "learning_rate": 5.92599822001666e-08,
979
+ "loss": 0.0001,
980
+ "reward": -3.0,
981
+ "reward_std": 0.0,
982
+ "rewards/env_reward_fn": -2.0,
983
+ "rewards/format_reward_fn": -1.0,
984
+ "step": 75
985
+ },
986
+ {
987
+ "completion_length": 197.625,
988
+ "epoch": 7.6,
989
+ "grad_norm": 0.050625696778297424,
990
+ "kl": 0.0035191430361010134,
991
+ "learning_rate": 3.798061746947995e-08,
992
+ "loss": 0.0001,
993
+ "reward": -3.0,
994
+ "reward_std": 0.0,
995
+ "rewards/env_reward_fn": -2.0,
996
+ "rewards/format_reward_fn": -1.0,
997
+ "step": 76
998
+ },
999
+ {
1000
+ "completion_length": 133.625,
1001
+ "epoch": 7.7,
1002
+ "grad_norm": 0.314410537481308,
1003
+ "kl": 0.0014756647869944572,
1004
+ "learning_rate": 2.1387846565474047e-08,
1005
+ "loss": 0.0001,
1006
+ "reward": -1.6187500059604645,
1007
+ "reward_std": 0.364735871553421,
1008
+ "rewards/env_reward_fn": -1.556249976158142,
1009
+ "rewards/format_reward_fn": -0.0625,
1010
+ "step": 77
1011
+ },
1012
+ {
1013
+ "completion_length": 240.5,
1014
+ "epoch": 7.8,
1015
+ "grad_norm": 0.03155818209052086,
1016
+ "kl": 0.002269520948175341,
1017
+ "learning_rate": 9.513254770636138e-09,
1018
+ "loss": 0.0001,
1019
+ "reward": -3.0,
1020
+ "reward_std": 0.0,
1021
+ "rewards/env_reward_fn": -2.0,
1022
+ "rewards/format_reward_fn": -1.0,
1023
+ "step": 78
1024
+ },
1025
+ {
1026
+ "completion_length": 240.9375,
1027
+ "epoch": 7.9,
1028
+ "grad_norm": 0.04115777835249901,
1029
+ "kl": 0.0034398152492940426,
1030
+ "learning_rate": 2.3794460453555046e-09,
1031
+ "loss": 0.0001,
1032
+ "reward": -3.0,
1033
+ "reward_std": 0.0,
1034
+ "rewards/env_reward_fn": -2.0,
1035
+ "rewards/format_reward_fn": -1.0,
1036
+ "step": 79
1037
+ },
1038
+ {
1039
+ "completion_length": 241.375,
1040
+ "epoch": 8.0,
1041
+ "grad_norm": 0.0072058564983308315,
1042
+ "kl": 0.0009698635112727061,
1043
+ "learning_rate": 0.0,
1044
+ "loss": 0.0,
1045
+ "reward": -3.0,
1046
+ "reward_std": 0.0,
1047
+ "rewards/env_reward_fn": -2.0,
1048
+ "rewards/format_reward_fn": -1.0,
1049
+ "step": 80
1050
+ }
1051
+ ],
1052
+ "logging_steps": 1,
1053
+ "max_steps": 80,
1054
+ "num_input_tokens_seen": 0,
1055
+ "num_train_epochs": 8,
1056
+ "save_steps": 999999,
1057
+ "stateful_callbacks": {
1058
+ "TrainerControl": {
1059
+ "args": {
1060
+ "should_epoch_stop": false,
1061
+ "should_evaluate": false,
1062
+ "should_log": false,
1063
+ "should_save": true,
1064
+ "should_training_stop": true
1065
+ },
1066
+ "attributes": {}
1067
+ }
1068
+ },
1069
+ "total_flos": 0.0,
1070
+ "train_batch_size": 2,
1071
+ "trial_name": null,
1072
+ "trial_params": null
1073
+ }
runs/grpo-20260426-062846/trainer/checkpoint-80/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca03ce83ecdd829c2eeee15e204f0c6e7a2afe2a7dd53071e53c5f9e4d4c0f8f
3
+ size 6033
runs/grpo-20260426-062846/trainer/checkpoint-80/vocab.json ADDED
The diff for this file is too large to render. See raw diff