Hugging Face's logo

Siddeshwar1625
/
osint-checkpoints

Safetensors
Model card Files
xet
 Community
Siddeshwar1625 commited on
Commit
0343b4e
·
verified ·
1 Parent(s): eb07223

Intermediate checkpoint upload step=30 (generator_train)

Browse files
Files changed (12) hide show
  1. .gitattributes +1 -0
  2. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/chat_template.jinja +54 -0
  3. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/config.json +57 -0
  4. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/generation_config.json +13 -0
  5. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/model.safetensors +3 -0
  6. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/optimizer.pt +3 -0
  7. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/rng_state.pth +3 -0
  8. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/scheduler.pt +3 -0
  9. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/tokenizer.json +3 -0
  10. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/tokenizer_config.json +32 -0
  11. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/trainer_state.json +588 -0
  12. self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/training_args.bin +3 -0
.gitattributes CHANGED
@@ -82,3 +82,4 @@ self_play_hf_l40s_full/round_004/answerer_train/checkpoint-60/tokenizer.json fil
82
  self_play_hf_l40s_full/round_004/answerer_train/checkpoint-90/tokenizer.json filter=lfs diff=lfs merge=lfs -text
83
  self_play_hf_l40s_full/round_004/answerer_train/checkpoint-120/tokenizer.json filter=lfs diff=lfs merge=lfs -text
84
  self_play_hf_l40s_full/round_004/answerer_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
82
  self_play_hf_l40s_full/round_004/answerer_train/checkpoint-90/tokenizer.json filter=lfs diff=lfs merge=lfs -text
83
  self_play_hf_l40s_full/round_004/answerer_train/checkpoint-120/tokenizer.json filter=lfs diff=lfs merge=lfs -text
84
  self_play_hf_l40s_full/round_004/answerer_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
85
+ self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": null,
7
+ "dtype": "float32",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 896,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4864,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention"
38
+ ],
39
+ "max_position_embeddings": 32768,
40
+ "max_window_layers": 21,
41
+ "model_type": "qwen2",
42
+ "num_attention_heads": 14,
43
+ "num_hidden_layers": 24,
44
+ "num_key_value_heads": 2,
45
+ "pad_token_id": 151643,
46
+ "rms_norm_eps": 1e-06,
47
+ "rope_parameters": {
48
+ "rope_theta": 1000000.0,
49
+ "rope_type": "default"
50
+ },
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.6.2",
54
+ "use_cache": false,
55
+ "use_sliding_window": false,
56
+ "vocab_size": 151936
57
+ }
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "repetition_penalty": 1.1,
9
+ "temperature": 0.7,
10
+ "top_k": 20,
11
+ "top_p": 0.8,
12
+ "transformers_version": "5.6.2"
13
+ }
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d6c71c39b53f29949105cd237c35478754dc9a1d3da086222d56f820208c264
3
+ size 1976163472
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:955f4ae455ecb91bae06fb4bfe413a5aae0fe7d0aecca2d176b51eb96232d584
3
+ size 3952509771
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41dae3129454fcbd2a668a22abc971454e5e6c939e1a5867ff90c2b866586678
3
+ size 14645
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1d413e3e3c82c1b74871c9cb9e959551000a6f2cc680c7750a2f6acda5c43d5
3
+ size 1465
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
3
+ size 11421892
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": true,
24
+ "local_files_only": false,
25
+ "model_max_length": 131072,
26
+ "pad_token": "<|endoftext|>",
27
+ "padding_side": "left",
28
+ "split_special_tokens": false,
29
+ "tokenizer_class": "Qwen2Tokenizer",
30
+ "truncation_side": "left",
31
+ "unk_token": null
32
+ }
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/trainer_state.json ADDED
@@ -0,0 +1,588 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.25,
6
+ "eval_steps": 500,
7
+ "global_step": 30,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 1.0,
19
+ "completions/max_length": 384.0,
20
+ "completions/max_terminated_length": 0.0,
21
+ "completions/mean_length": 384.0,
22
+ "completions/mean_terminated_length": 0.0,
23
+ "completions/min_length": 384.0,
24
+ "completions/min_terminated_length": 0.0,
25
+ "entropy": 4.218832492828369,
26
+ "epoch": 0.041666666666666664,
27
+ "frac_reward_zero_std": 0.5,
28
+ "grad_norm": 2.3453547954559326,
29
+ "kl": 0.0003995061560999602,
30
+ "learning_rate": 5e-06,
31
+ "loss": 0.062480177730321884,
32
+ "num_tokens": 25296.0,
33
+ "reward": -0.16249999403953552,
34
+ "reward_std": 0.24186772108078003,
35
+ "rewards/GeneratorRewardFunction/mean": -0.16249999403953552,
36
+ "rewards/GeneratorRewardFunction/std": 0.24186773598194122,
37
+ "step": 1,
38
+ "step_time": 12.200799825999638
39
+ },
40
+ {
41
+ "clip_ratio/high_max": 0.0,
42
+ "clip_ratio/high_mean": 0.0,
43
+ "clip_ratio/low_mean": 0.0325520820915699,
44
+ "clip_ratio/low_min": 0.0325520820915699,
45
+ "clip_ratio/region_mean": 0.0325520820915699,
46
+ "entropy": 4.258089542388916,
47
+ "epoch": 0.08333333333333333,
48
+ "grad_norm": 1.9494712352752686,
49
+ "kl": 0.01056413259357214,
50
+ "learning_rate": 4.958333333333334e-06,
51
+ "loss": 0.10853039473295212,
52
+ "step": 2,
53
+ "step_time": 0.22676954300004581
54
+ },
55
+ {
56
+ "clip_ratio/high_max": 0.0618489570915699,
57
+ "clip_ratio/high_mean": 0.0618489570915699,
58
+ "clip_ratio/low_mean": 0.0,
59
+ "clip_ratio/low_min": 0.0,
60
+ "clip_ratio/region_mean": 0.0618489570915699,
61
+ "entropy": 3.910761594772339,
62
+ "epoch": 0.125,
63
+ "grad_norm": 1.5771934986114502,
64
+ "kl": 0.03891553357243538,
65
+ "learning_rate": 4.9166666666666665e-06,
66
+ "loss": -0.08321236819028854,
67
+ "step": 3,
68
+ "step_time": 0.2259805369994865
69
+ },
70
+ {
71
+ "clip_ratio/high_max": 0.0755208358168602,
72
+ "clip_ratio/high_mean": 0.0755208358168602,
73
+ "clip_ratio/low_mean": 0.0,
74
+ "clip_ratio/low_min": 0.0,
75
+ "clip_ratio/region_mean": 0.0755208358168602,
76
+ "entropy": 4.357999324798584,
77
+ "epoch": 0.16666666666666666,
78
+ "grad_norm": 1.4966953992843628,
79
+ "kl": 0.03107132576406002,
80
+ "learning_rate": 4.875e-06,
81
+ "loss": -0.08200164884328842,
82
+ "step": 4,
83
+ "step_time": 0.22625164300006873
84
+ },
85
+ {
86
+ "clip_ratio/high_max": 0.0,
87
+ "clip_ratio/high_mean": 0.0,
88
+ "clip_ratio/low_mean": 0.0,
89
+ "clip_ratio/low_min": 0.0,
90
+ "clip_ratio/region_mean": 0.0,
91
+ "completions/clipped_ratio": 1.0,
92
+ "completions/max_length": 384.0,
93
+ "completions/max_terminated_length": 0.0,
94
+ "completions/mean_length": 384.0,
95
+ "completions/mean_terminated_length": 0.0,
96
+ "completions/min_length": 384.0,
97
+ "completions/min_terminated_length": 0.0,
98
+ "entropy": 3.2972638607025146,
99
+ "epoch": 0.20833333333333334,
100
+ "frac_reward_zero_std": 1.0,
101
+ "grad_norm": 0.012044131755828857,
102
+ "kl": 0.021904518827795982,
103
+ "learning_rate": 4.833333333333333e-06,
104
+ "loss": 5.476129081216641e-05,
105
+ "num_tokens": 50928.0,
106
+ "reward": -0.05000000074505806,
107
+ "reward_std": 0.0,
108
+ "rewards/GeneratorRewardFunction/mean": -0.05000000074505806,
109
+ "rewards/GeneratorRewardFunction/std": 0.0,
110
+ "step": 5,
111
+ "step_time": 12.42558103400006
112
+ },
113
+ {
114
+ "clip_ratio/high_max": 0.0,
115
+ "clip_ratio/high_mean": 0.0,
116
+ "clip_ratio/low_mean": 0.0,
117
+ "clip_ratio/low_min": 0.0,
118
+ "clip_ratio/region_mean": 0.0,
119
+ "entropy": 4.389660358428955,
120
+ "epoch": 0.25,
121
+ "grad_norm": 0.01742626540362835,
122
+ "kl": 0.034423865377902985,
123
+ "learning_rate": 4.791666666666668e-06,
124
+ "loss": 8.605967741459608e-05,
125
+ "step": 6,
126
+ "step_time": 0.2294949209999686
127
+ },
128
+ {
129
+ "clip_ratio/high_max": 0.0,
130
+ "clip_ratio/high_mean": 0.0,
131
+ "clip_ratio/low_mean": 0.0,
132
+ "clip_ratio/low_min": 0.0,
133
+ "clip_ratio/region_mean": 0.0,
134
+ "entropy": 4.004185199737549,
135
+ "epoch": 0.2916666666666667,
136
+ "grad_norm": 0.018903816118836403,
137
+ "kl": 0.0423373244702816,
138
+ "learning_rate": 4.75e-06,
139
+ "loss": 0.000105843304481823,
140
+ "step": 7,
141
+ "step_time": 0.22864662399933877
142
+ },
143
+ {
144
+ "clip_ratio/high_max": 0.0,
145
+ "clip_ratio/high_mean": 0.0,
146
+ "clip_ratio/low_mean": 0.0,
147
+ "clip_ratio/low_min": 0.0,
148
+ "clip_ratio/region_mean": 0.0,
149
+ "entropy": 4.172105312347412,
150
+ "epoch": 0.3333333333333333,
151
+ "grad_norm": 0.02683727815747261,
152
+ "kl": 0.05335487797856331,
153
+ "learning_rate": 4.708333333333334e-06,
154
+ "loss": 0.00013338720600586385,
155
+ "step": 8,
156
+ "step_time": 0.22813066999970033
157
+ },
158
+ {
159
+ "clip_ratio/high_max": 0.0,
160
+ "clip_ratio/high_mean": 0.0,
161
+ "clip_ratio/low_mean": 0.0,
162
+ "clip_ratio/low_min": 0.0,
163
+ "clip_ratio/region_mean": 0.0,
164
+ "completions/clipped_ratio": 1.0,
165
+ "completions/max_length": 384.0,
166
+ "completions/max_terminated_length": 0.0,
167
+ "completions/mean_length": 384.0,
168
+ "completions/mean_terminated_length": 0.0,
169
+ "completions/min_length": 384.0,
170
+ "completions/min_terminated_length": 0.0,
171
+ "entropy": 3.9359867572784424,
172
+ "epoch": 0.375,
173
+ "frac_reward_zero_std": 0.75,
174
+ "grad_norm": 0.7581679821014404,
175
+ "kl": 0.05953861400485039,
176
+ "learning_rate": 4.666666666666667e-06,
177
+ "loss": -0.031090745702385902,
178
+ "num_tokens": 76192.0,
179
+ "reward": -0.08749999850988388,
180
+ "reward_std": 0.14999999105930328,
181
+ "rewards/GeneratorRewardFunction/mean": -0.08749999850988388,
182
+ "rewards/GeneratorRewardFunction/std": 0.15000000596046448,
183
+ "step": 9,
184
+ "step_time": 12.354528401000607
185
+ },
186
+ {
187
+ "clip_ratio/high_max": 0.0006510416860692203,
188
+ "clip_ratio/high_mean": 0.0006510416860692203,
189
+ "clip_ratio/low_mean": 0.0,
190
+ "clip_ratio/low_min": 0.0,
191
+ "clip_ratio/region_mean": 0.0006510416860692203,
192
+ "entropy": 3.8035316467285156,
193
+ "epoch": 0.4166666666666667,
194
+ "grad_norm": 0.7192441821098328,
195
+ "kl": 0.05113422870635986,
196
+ "learning_rate": 4.625000000000001e-06,
197
+ "loss": -0.031105250120162964,
198
+ "step": 10,
199
+ "step_time": 0.22121003899974312
200
+ },
201
+ {
202
+ "clip_ratio/high_max": 0.0,
203
+ "clip_ratio/high_mean": 0.0,
204
+ "clip_ratio/low_mean": 0.0,
205
+ "clip_ratio/low_min": 0.0,
206
+ "clip_ratio/region_mean": 0.0,
207
+ "entropy": 4.490121841430664,
208
+ "epoch": 0.4583333333333333,
209
+ "grad_norm": 0.020974429324269295,
210
+ "kl": 0.06544887274503708,
211
+ "learning_rate": 4.583333333333333e-06,
212
+ "loss": 0.00016362218593712896,
213
+ "step": 11,
214
+ "step_time": 0.22179323100044712
215
+ },
216
+ {
217
+ "clip_ratio/high_max": 0.01692708395421505,
218
+ "clip_ratio/high_mean": 0.01692708395421505,
219
+ "clip_ratio/low_mean": 0.010416666977107525,
220
+ "clip_ratio/low_min": 0.010416666977107525,
221
+ "clip_ratio/region_mean": 0.02734375,
222
+ "entropy": 4.094717025756836,
223
+ "epoch": 0.5,
224
+ "grad_norm": 2.440980911254883,
225
+ "kl": 0.056731849908828735,
226
+ "learning_rate": 4.541666666666667e-06,
227
+ "loss": 0.06417856365442276,
228
+ "step": 12,
229
+ "step_time": 0.22233516199958103
230
+ },
231
+ {
232
+ "clip_ratio/high_max": 0.0,
233
+ "clip_ratio/high_mean": 0.0,
234
+ "clip_ratio/low_mean": 0.0,
235
+ "clip_ratio/low_min": 0.0,
236
+ "clip_ratio/region_mean": 0.0,
237
+ "completions/clipped_ratio": 1.0,
238
+ "completions/max_length": 384.0,
239
+ "completions/max_terminated_length": 0.0,
240
+ "completions/mean_length": 384.0,
241
+ "completions/mean_terminated_length": 0.0,
242
+ "completions/min_length": 384.0,
243
+ "completions/min_terminated_length": 0.0,
244
+ "entropy": 4.305630207061768,
245
+ "epoch": 0.5416666666666666,
246
+ "frac_reward_zero_std": 0.75,
247
+ "grad_norm": 0.7685584425926208,
248
+ "kl": 0.06827350705862045,
249
+ "learning_rate": 4.5e-06,
250
+ "loss": -0.03110312856733799,
251
+ "num_tokens": 101896.0,
252
+ "reward": -0.05656249821186066,
253
+ "reward_std": 0.026250001043081284,
254
+ "rewards/GeneratorRewardFunction/mean": -0.05656249821186066,
255
+ "rewards/GeneratorRewardFunction/std": 0.026250001043081284,
256
+ "step": 13,
257
+ "step_time": 12.351281250999818
258
+ },
259
+ {
260
+ "clip_ratio/high_max": 0.0006510416860692203,
261
+ "clip_ratio/high_mean": 0.0006510416860692203,
262
+ "clip_ratio/low_mean": 0.0,
263
+ "clip_ratio/low_min": 0.0,
264
+ "clip_ratio/region_mean": 0.0006510416860692203,
265
+ "entropy": 4.384155750274658,
266
+ "epoch": 0.5833333333333334,
267
+ "grad_norm": 1.074666976928711,
268
+ "kl": 0.09433463960886002,
269
+ "learning_rate": 4.4583333333333336e-06,
270
+ "loss": -0.06210761144757271,
271
+ "step": 14,
272
+ "step_time": 0.228220768999563
273
+ },
274
+ {
275
+ "clip_ratio/high_max": 0.0,
276
+ "clip_ratio/high_mean": 0.0,
277
+ "clip_ratio/low_mean": 0.0013020833721384406,
278
+ "clip_ratio/low_min": 0.0013020833721384406,
279
+ "clip_ratio/region_mean": 0.0013020833721384406,
280
+ "entropy": 4.467245578765869,
281
+ "epoch": 0.625,
282
+ "grad_norm": 2.4682021141052246,
283
+ "kl": 0.0888032540678978,
284
+ "learning_rate": 4.416666666666667e-06,
285
+ "loss": 0.09433958679437637,
286
+ "step": 15,
287
+ "step_time": 0.22796953400029452
288
+ },
289
+ {
290
+ "clip_ratio/high_max": 0.0,
291
+ "clip_ratio/high_mean": 0.0,
292
+ "clip_ratio/low_mean": 0.0,
293
+ "clip_ratio/low_min": 0.0,
294
+ "clip_ratio/region_mean": 0.0,
295
+ "entropy": 4.379356384277344,
296
+ "epoch": 0.6666666666666666,
297
+ "grad_norm": 0.05159206688404083,
298
+ "kl": 0.09723836183547974,
299
+ "learning_rate": 4.3750000000000005e-06,
300
+ "loss": 0.0002430959139019251,
301
+ "step": 16,
302
+ "step_time": 0.22781476000000112
303
+ },
304
+ {
305
+ "clip_ratio/high_max": 0.0,
306
+ "clip_ratio/high_mean": 0.0,
307
+ "clip_ratio/low_mean": 0.0,
308
+ "clip_ratio/low_min": 0.0,
309
+ "clip_ratio/region_mean": 0.0,
310
+ "completions/clipped_ratio": 1.0,
311
+ "completions/max_length": 384.0,
312
+ "completions/max_terminated_length": 0.0,
313
+ "completions/mean_length": 384.0,
314
+ "completions/mean_terminated_length": 0.0,
315
+ "completions/min_length": 384.0,
316
+ "completions/min_terminated_length": 0.0,
317
+ "entropy": 4.611701965332031,
318
+ "epoch": 0.7083333333333334,
319
+ "frac_reward_zero_std": 1.0,
320
+ "grad_norm": 0.027016103267669678,
321
+ "kl": 0.09803465753793716,
322
+ "learning_rate": 4.333333333333334e-06,
323
+ "loss": 0.00024508664500899613,
324
+ "num_tokens": 127124.0,
325
+ "reward": -0.05000000074505806,
326
+ "reward_std": 0.0,
327
+ "rewards/GeneratorRewardFunction/mean": -0.05000000074505806,
328
+ "rewards/GeneratorRewardFunction/std": 0.0,
329
+ "step": 17,
330
+ "step_time": 12.34623024100074
331
+ },
332
+ {
333
+ "clip_ratio/high_max": 0.0,
334
+ "clip_ratio/high_mean": 0.0,
335
+ "clip_ratio/low_mean": 0.0,
336
+ "clip_ratio/low_min": 0.0,
337
+ "clip_ratio/region_mean": 0.0,
338
+ "entropy": 4.3698506355285645,
339
+ "epoch": 0.75,
340
+ "grad_norm": 0.027759015560150146,
341
+ "kl": 0.10794633626937866,
342
+ "learning_rate": 4.2916666666666665e-06,
343
+ "loss": 0.00026986582088284194,
344
+ "step": 18,
345
+ "step_time": 0.22221811200051889
346
+ },
347
+ {
348
+ "clip_ratio/high_max": 0.0,
349
+ "clip_ratio/high_mean": 0.0,
350
+ "clip_ratio/low_mean": 0.0,
351
+ "clip_ratio/low_min": 0.0,
352
+ "clip_ratio/region_mean": 0.0,
353
+ "entropy": 4.589417934417725,
354
+ "epoch": 0.7916666666666666,
355
+ "grad_norm": 0.033771730959415436,
356
+ "kl": 0.1163717731833458,
357
+ "learning_rate": 4.25e-06,
358
+ "loss": 0.0002909294271375984,
359
+ "step": 19,
360
+ "step_time": 0.22278399300012097
361
+ },
362
+ {
363
+ "clip_ratio/high_max": 0.0,
364
+ "clip_ratio/high_mean": 0.0,
365
+ "clip_ratio/low_mean": 0.0,
366
+ "clip_ratio/low_min": 0.0,
367
+ "clip_ratio/region_mean": 0.0,
368
+ "entropy": 4.381205081939697,
369
+ "epoch": 0.8333333333333334,
370
+ "grad_norm": 0.034088168293237686,
371
+ "kl": 0.12577062845230103,
372
+ "learning_rate": 4.208333333333333e-06,
373
+ "loss": 0.0003144265792798251,
374
+ "step": 20,
375
+ "step_time": 0.2227981229998477
376
+ },
377
+ {
378
+ "clip_ratio/high_max": 0.0,
379
+ "clip_ratio/high_mean": 0.0,
380
+ "clip_ratio/low_mean": 0.0,
381
+ "clip_ratio/low_min": 0.0,
382
+ "clip_ratio/region_mean": 0.0,
383
+ "completions/clipped_ratio": 1.0,
384
+ "completions/max_length": 384.0,
385
+ "completions/max_terminated_length": 0.0,
386
+ "completions/mean_length": 384.0,
387
+ "completions/mean_terminated_length": 0.0,
388
+ "completions/min_length": 384.0,
389
+ "completions/min_terminated_length": 0.0,
390
+ "entropy": 4.56951904296875,
391
+ "epoch": 0.875,
392
+ "frac_reward_zero_std": 0.75,
393
+ "grad_norm": 0.030865877866744995,
394
+ "kl": 0.1258154958486557,
395
+ "learning_rate": 4.166666666666667e-06,
396
+ "loss": 0.00031453874544240534,
397
+ "num_tokens": 152420.0,
398
+ "reward": -0.06875000149011612,
399
+ "reward_std": 0.07500000298023224,
400
+ "rewards/GeneratorRewardFunction/mean": -0.06875000149011612,
401
+ "rewards/GeneratorRewardFunction/std": 0.07500000298023224,
402
+ "step": 21,
403
+ "step_time": 12.431868073999794
404
+ },
405
+ {
406
+ "clip_ratio/high_max": 0.0006510416860692203,
407
+ "clip_ratio/high_mean": 0.0006510416860692203,
408
+ "clip_ratio/low_mean": 0.0,
409
+ "clip_ratio/low_min": 0.0,
410
+ "clip_ratio/region_mean": 0.0006510416860692203,
411
+ "entropy": 4.256842136383057,
412
+ "epoch": 0.9166666666666666,
413
+ "grad_norm": 1.0559728145599365,
414
+ "kl": 0.11962169408798218,
415
+ "learning_rate": 4.125e-06,
416
+ "loss": -0.06200842559337616,
417
+ "step": 22,
418
+ "step_time": 0.22747174799951608
419
+ },
420
+ {
421
+ "clip_ratio/high_max": 0.0,
422
+ "clip_ratio/high_mean": 0.0,
423
+ "clip_ratio/low_mean": 0.0,
424
+ "clip_ratio/low_min": 0.0,
425
+ "clip_ratio/region_mean": 0.0,
426
+ "entropy": 4.441510200500488,
427
+ "epoch": 0.9583333333333334,
428
+ "grad_norm": 0.8300934433937073,
429
+ "kl": 0.12788282334804535,
430
+ "learning_rate": 4.083333333333334e-06,
431
+ "loss": -0.030932163819670677,
432
+ "step": 23,
433
+ "step_time": 0.2274885289998565
434
+ },
435
+ {
436
+ "clip_ratio/high_max": 0.0,
437
+ "clip_ratio/high_mean": 0.0,
438
+ "clip_ratio/low_mean": 0.0026041667442768812,
439
+ "clip_ratio/low_min": 0.0026041667442768812,
440
+ "clip_ratio/region_mean": 0.0026041667442768812,
441
+ "entropy": 4.461675643920898,
442
+ "epoch": 1.0,
443
+ "grad_norm": 2.326770067214966,
444
+ "kl": 0.1365349292755127,
445
+ "learning_rate": 4.041666666666667e-06,
446
+ "loss": 0.09416845440864563,
447
+ "step": 24,
448
+ "step_time": 0.22709047199987253
449
+ },
450
+ {
451
+ "clip_ratio/high_max": 0.0,
452
+ "clip_ratio/high_mean": 0.0,
453
+ "clip_ratio/low_mean": 0.0,
454
+ "clip_ratio/low_min": 0.0,
455
+ "clip_ratio/region_mean": 0.0,
456
+ "completions/clipped_ratio": 1.0,
457
+ "completions/max_length": 384.0,
458
+ "completions/max_terminated_length": 0.0,
459
+ "completions/mean_length": 384.0,
460
+ "completions/mean_terminated_length": 0.0,
461
+ "completions/min_length": 384.0,
462
+ "completions/min_terminated_length": 0.0,
463
+ "entropy": 4.375850200653076,
464
+ "epoch": 1.0416666666666667,
465
+ "frac_reward_zero_std": 1.0,
466
+ "grad_norm": 0.031697701662778854,
467
+ "kl": 0.1358719766139984,
468
+ "learning_rate": 4.000000000000001e-06,
469
+ "loss": 0.0003396799729671329,
470
+ "num_tokens": 177592.0,
471
+ "reward": -0.05000000074505806,
472
+ "reward_std": 0.0,
473
+ "rewards/GeneratorRewardFunction/mean": -0.05000000074505806,
474
+ "rewards/GeneratorRewardFunction/std": 0.0,
475
+ "step": 25,
476
+ "step_time": 12.350898434000555
477
+ },
478
+ {
479
+ "clip_ratio/high_max": 0.0,
480
+ "clip_ratio/high_mean": 0.0,
481
+ "clip_ratio/low_mean": 0.0,
482
+ "clip_ratio/low_min": 0.0,
483
+ "clip_ratio/region_mean": 0.0,
484
+ "entropy": 4.472981929779053,
485
+ "epoch": 1.0833333333333333,
486
+ "grad_norm": 0.033567290753126144,
487
+ "kl": 0.1449340432882309,
488
+ "learning_rate": 3.958333333333333e-06,
489
+ "loss": 0.0003623350930865854,
490
+ "step": 26,
491
+ "step_time": 0.22230296500038094
492
+ },
493
+ {
494
+ "clip_ratio/high_max": 0.0,
495
+ "clip_ratio/high_mean": 0.0,
496
+ "clip_ratio/low_mean": 0.0,
497
+ "clip_ratio/low_min": 0.0,
498
+ "clip_ratio/region_mean": 0.0,
499
+ "entropy": 4.471365451812744,
500
+ "epoch": 1.125,
501
+ "grad_norm": 0.033745091408491135,
502
+ "kl": 0.14426572620868683,
503
+ "learning_rate": 3.916666666666667e-06,
504
+ "loss": 0.0003606643294915557,
505
+ "step": 27,
506
+ "step_time": 0.22167697199984104
507
+ },
508
+ {
509
+ "clip_ratio/high_max": 0.0,
510
+ "clip_ratio/high_mean": 0.0,
511
+ "clip_ratio/low_mean": 0.0,
512
+ "clip_ratio/low_min": 0.0,
513
+ "clip_ratio/region_mean": 0.0,
514
+ "entropy": 4.190944671630859,
515
+ "epoch": 1.1666666666666667,
516
+ "grad_norm": 0.03441477566957474,
517
+ "kl": 0.14298784732818604,
518
+ "learning_rate": 3.875e-06,
519
+ "loss": 0.0003574696311261505,
520
+ "step": 28,
521
+ "step_time": 0.22162332900006732
522
+ },
523
+ {
524
+ "clip_ratio/high_max": 0.0,
525
+ "clip_ratio/high_mean": 0.0,
526
+ "clip_ratio/low_mean": 0.0,
527
+ "clip_ratio/low_min": 0.0,
528
+ "clip_ratio/region_mean": 0.0,
529
+ "completions/clipped_ratio": 1.0,
530
+ "completions/max_length": 384.0,
531
+ "completions/max_terminated_length": 0.0,
532
+ "completions/mean_length": 384.0,
533
+ "completions/mean_terminated_length": 0.0,
534
+ "completions/min_length": 384.0,
535
+ "completions/min_terminated_length": 0.0,
536
+ "entropy": 4.321583271026611,
537
+ "epoch": 1.2083333333333333,
538
+ "frac_reward_zero_std": 0.75,
539
+ "grad_norm": 2.1965482234954834,
540
+ "kl": 0.13321949541568756,
541
+ "learning_rate": 3.833333333333334e-06,
542
+ "loss": 0.09402058273553848,
543
+ "num_tokens": 203376.0,
544
+ "reward": -0.06875000149011612,
545
+ "reward_std": 0.07499999552965164,
546
+ "rewards/GeneratorRewardFunction/mean": -0.06875000149011612,
547
+ "rewards/GeneratorRewardFunction/std": 0.07500000298023224,
548
+ "step": 29,
549
+ "step_time": 12.401389794999886
550
+ },
551
+ {
552
+ "clip_ratio/high_max": 0.0,
553
+ "clip_ratio/high_mean": 0.0,
554
+ "clip_ratio/low_mean": 0.0,
555
+ "clip_ratio/low_min": 0.0,
556
+ "clip_ratio/region_mean": 0.0,
557
+ "entropy": 4.206346035003662,
558
+ "epoch": 1.25,
559
+ "grad_norm": 0.8052829504013062,
560
+ "kl": 0.14546459913253784,
561
+ "learning_rate": 3.7916666666666666e-06,
562
+ "loss": -0.03093709610402584,
563
+ "step": 30,
564
+ "step_time": 0.22923127000012755
565
+ }
566
+ ],
567
+ "logging_steps": 1,
568
+ "max_steps": 120,
569
+ "num_input_tokens_seen": 203376,
570
+ "num_train_epochs": 5,
571
+ "save_steps": 30,
572
+ "stateful_callbacks": {
573
+ "TrainerControl": {
574
+ "args": {
575
+ "should_epoch_stop": false,
576
+ "should_evaluate": false,
577
+ "should_log": false,
578
+ "should_save": true,
579
+ "should_training_stop": false
580
+ },
581
+ "attributes": {}
582
+ }
583
+ },
584
+ "total_flos": 0.0,
585
+ "train_batch_size": 4,
586
+ "trial_name": null,
587
+ "trial_params": null
588
+ }
self_play_hf_l40s_full/round_005/generator_train/checkpoint-30/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1afbfa894ee2b20e77f8fc01bf94ad1916f3036851a51f84658b8d193a779a9b
3
+ size 7249