Hugging Face's logo

Siddeshwar1625
/
osint-checkpoints

Safetensors
Model card Files
xet
 Community
Siddeshwar1625 commited on
Commit
181f60b
·
verified ·
1 Parent(s): 2fd9129

Intermediate checkpoint upload step=60 (answerer_train)

Browse files
Files changed (12) hide show
  1. .gitattributes +1 -0
  2. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/chat_template.jinja +54 -0
  3. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/config.json +57 -0
  4. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/generation_config.json +13 -0
  5. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/model.safetensors +3 -0
  6. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/optimizer.pt +3 -0
  7. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/rng_state.pth +3 -0
  8. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/scheduler.pt +3 -0
  9. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/tokenizer.json +3 -0
  10. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/tokenizer_config.json +32 -0
  11. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/trainer_state.json +1129 -0
  12. self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/training_args.bin +3 -0
.gitattributes CHANGED
@@ -48,3 +48,4 @@ self_play_hf_l40s_full/round_001/generator_train/checkpoint-90/tokenizer.json fi
48
  self_play_hf_l40s_full/round_001/generator_train/checkpoint-120/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
  self_play_hf_l40s_full/round_001/generator_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
50
  self_play_hf_l40s_full/round_001/answerer_train/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
48
  self_play_hf_l40s_full/round_001/generator_train/checkpoint-120/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
  self_play_hf_l40s_full/round_001/generator_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
50
  self_play_hf_l40s_full/round_001/answerer_train/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
51
+ self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/tokenizer.json filter=lfs diff=lfs merge=lfs -text
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": null,
7
+ "dtype": "float32",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 896,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4864,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention"
38
+ ],
39
+ "max_position_embeddings": 32768,
40
+ "max_window_layers": 21,
41
+ "model_type": "qwen2",
42
+ "num_attention_heads": 14,
43
+ "num_hidden_layers": 24,
44
+ "num_key_value_heads": 2,
45
+ "pad_token_id": 151643,
46
+ "rms_norm_eps": 1e-06,
47
+ "rope_parameters": {
48
+ "rope_theta": 1000000.0,
49
+ "rope_type": "default"
50
+ },
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.6.2",
54
+ "use_cache": false,
55
+ "use_sliding_window": false,
56
+ "vocab_size": 151936
57
+ }
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "repetition_penalty": 1.1,
9
+ "temperature": 0.7,
10
+ "top_k": 20,
11
+ "top_p": 0.8,
12
+ "transformers_version": "5.6.2"
13
+ }
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:593ee6cbbd89736dd91e85159352b12cfc69bbe64abbfcad73f788ad6532a8ca
3
+ size 1976163472
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:363e9a2654a4dbd5d0b2a4f9c7e3c0876bbd295f6afa9a45fa0f323dc17c8b93
3
+ size 3952509771
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2258c8f7e6687f5fa2dcc6cd5ebbafa0a88234fdbe6ce45ab0d52ed8947a2e83
3
+ size 14645
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aab883bcab393f0c916048be23a6c0284269524eb2950c4249e03acc7cac4f75
3
+ size 1465
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
3
+ size 11421892
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": true,
24
+ "local_files_only": false,
25
+ "model_max_length": 131072,
26
+ "pad_token": "<|endoftext|>",
27
+ "padding_side": "left",
28
+ "split_special_tokens": false,
29
+ "tokenizer_class": "Qwen2Tokenizer",
30
+ "truncation_side": "left",
31
+ "unk_token": null
32
+ }
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/trainer_state.json ADDED
@@ -0,0 +1,1129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.5,
6
+ "eval_steps": 500,
7
+ "global_step": 60,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 1.0,
19
+ "completions/max_length": 256.0,
20
+ "completions/max_terminated_length": 0.0,
21
+ "completions/mean_length": 256.0,
22
+ "completions/mean_terminated_length": 0.0,
23
+ "completions/min_length": 256.0,
24
+ "completions/min_terminated_length": 0.0,
25
+ "entropy": 1.0711654424667358,
26
+ "epoch": 0.025,
27
+ "frac_reward_zero_std": 0.0,
28
+ "grad_norm": 2.947098731994629,
29
+ "kl": 0.0,
30
+ "learning_rate": 3e-06,
31
+ "loss": 0.06321408599615097,
32
+ "num_tokens": 9896.0,
33
+ "reward": 0.5316684246063232,
34
+ "reward_std": 0.11483359336853027,
35
+ "rewards/AnswererRewardFunction/mean": 0.5316684246063232,
36
+ "rewards/AnswererRewardFunction/std": 0.11483359336853027,
37
+ "step": 1,
38
+ "step_time": 8.027100405000056
39
+ },
40
+ {
41
+ "clip_ratio/high_max": 0.033203125,
42
+ "clip_ratio/high_mean": 0.033203125,
43
+ "clip_ratio/low_mean": 0.0224609375,
44
+ "clip_ratio/low_min": 0.0224609375,
45
+ "clip_ratio/region_mean": 0.0556640625,
46
+ "entropy": 0.8880369663238525,
47
+ "epoch": 0.05,
48
+ "grad_norm": 2.81626296043396,
49
+ "kl": 0.04245668277144432,
50
+ "learning_rate": 2.9750000000000003e-06,
51
+ "loss": -0.07583755254745483,
52
+ "step": 2,
53
+ "step_time": 0.12222773600001346
54
+ },
55
+ {
56
+ "clip_ratio/high_max": 0.0166015625,
57
+ "clip_ratio/high_mean": 0.0166015625,
58
+ "clip_ratio/low_mean": 0.05078125,
59
+ "clip_ratio/low_min": 0.05078125,
60
+ "clip_ratio/region_mean": 0.0673828125,
61
+ "entropy": 1.0095021724700928,
62
+ "epoch": 0.075,
63
+ "grad_norm": 5.157853603363037,
64
+ "kl": 0.016499709337949753,
65
+ "learning_rate": 2.9499999999999997e-06,
66
+ "loss": 0.18797507882118225,
67
+ "step": 3,
68
+ "step_time": 0.11191487599990069
69
+ },
70
+ {
71
+ "clip_ratio/high_max": 0.0546875,
72
+ "clip_ratio/high_mean": 0.0546875,
73
+ "clip_ratio/low_mean": 0.025390625,
74
+ "clip_ratio/low_min": 0.025390625,
75
+ "clip_ratio/region_mean": 0.080078125,
76
+ "entropy": 0.8205338716506958,
77
+ "epoch": 0.1,
78
+ "grad_norm": 4.075566291809082,
79
+ "kl": 0.05314125120639801,
80
+ "learning_rate": 2.925e-06,
81
+ "loss": -0.16264761984348297,
82
+ "step": 4,
83
+ "step_time": 0.11004377599988402
84
+ },
85
+ {
86
+ "clip_ratio/high_max": 0.0009765625,
87
+ "clip_ratio/high_mean": 0.0009765625,
88
+ "clip_ratio/low_mean": 0.0,
89
+ "clip_ratio/low_min": 0.0,
90
+ "clip_ratio/region_mean": 0.0009765625,
91
+ "completions/clipped_ratio": 1.0,
92
+ "completions/max_length": 256.0,
93
+ "completions/max_terminated_length": 0.0,
94
+ "completions/mean_length": 256.0,
95
+ "completions/mean_terminated_length": 0.0,
96
+ "completions/min_length": 256.0,
97
+ "completions/min_terminated_length": 0.0,
98
+ "entropy": 0.751447856426239,
99
+ "epoch": 0.125,
100
+ "frac_reward_zero_std": 0.0,
101
+ "grad_norm": 3.1406376361846924,
102
+ "kl": 0.021290192380547523,
103
+ "learning_rate": 2.9e-06,
104
+ "loss": -0.0005250873509794474,
105
+ "num_tokens": 19632.0,
106
+ "reward": 0.543575644493103,
107
+ "reward_std": 0.11543858796358109,
108
+ "rewards/AnswererRewardFunction/mean": 0.543575644493103,
109
+ "rewards/AnswererRewardFunction/std": 0.11543858796358109,
110
+ "step": 5,
111
+ "step_time": 8.046513188000063
112
+ },
113
+ {
114
+ "clip_ratio/high_max": 0.00390625,
115
+ "clip_ratio/high_mean": 0.00390625,
116
+ "clip_ratio/low_mean": 0.0078125,
117
+ "clip_ratio/low_min": 0.0078125,
118
+ "clip_ratio/region_mean": 0.01171875,
119
+ "entropy": 0.6940755248069763,
120
+ "epoch": 0.15,
121
+ "grad_norm": 3.9180352687835693,
122
+ "kl": 0.02370832860469818,
123
+ "learning_rate": 2.875e-06,
124
+ "loss": 0.061240844428539276,
125
+ "step": 6,
126
+ "step_time": 0.1118731530000332
127
+ },
128
+ {
129
+ "clip_ratio/high_max": 0.0087890625,
130
+ "clip_ratio/high_mean": 0.0087890625,
131
+ "clip_ratio/low_mean": 0.0234375,
132
+ "clip_ratio/low_min": 0.0234375,
133
+ "clip_ratio/region_mean": 0.0322265625,
134
+ "entropy": 0.8035241365432739,
135
+ "epoch": 0.175,
136
+ "grad_norm": 3.497161865234375,
137
+ "kl": 0.03351783752441406,
138
+ "learning_rate": 2.85e-06,
139
+ "loss": -0.004166277591139078,
140
+ "step": 7,
141
+ "step_time": 0.1107469599999149
142
+ },
143
+ {
144
+ "clip_ratio/high_max": 0.0263671875,
145
+ "clip_ratio/high_mean": 0.0263671875,
146
+ "clip_ratio/low_mean": 0.0068359375,
147
+ "clip_ratio/low_min": 0.0068359375,
148
+ "clip_ratio/region_mean": 0.033203125,
149
+ "entropy": 0.5180478096008301,
150
+ "epoch": 0.2,
151
+ "grad_norm": 1.883070945739746,
152
+ "kl": 0.04320289194583893,
153
+ "learning_rate": 2.825e-06,
154
+ "loss": -0.053033724427223206,
155
+ "step": 8,
156
+ "step_time": 0.11128661100019599
157
+ },
158
+ {
159
+ "clip_ratio/high_max": 0.0009765625,
160
+ "clip_ratio/high_mean": 0.0009765625,
161
+ "clip_ratio/low_mean": 0.0,
162
+ "clip_ratio/low_min": 0.0,
163
+ "clip_ratio/region_mean": 0.0009765625,
164
+ "completions/clipped_ratio": 1.0,
165
+ "completions/max_length": 256.0,
166
+ "completions/max_terminated_length": 0.0,
167
+ "completions/mean_length": 256.0,
168
+ "completions/mean_terminated_length": 0.0,
169
+ "completions/min_length": 256.0,
170
+ "completions/min_terminated_length": 0.0,
171
+ "entropy": 0.6499772071838379,
172
+ "epoch": 0.225,
173
+ "frac_reward_zero_std": 0.0,
174
+ "grad_norm": 3.706423759460449,
175
+ "kl": 0.041926056146621704,
176
+ "learning_rate": 2.8000000000000003e-06,
177
+ "loss": -0.12885785102844238,
178
+ "num_tokens": 29444.0,
179
+ "reward": 0.5444201231002808,
180
+ "reward_std": 0.1789802759885788,
181
+ "rewards/AnswererRewardFunction/mean": 0.5444201231002808,
182
+ "rewards/AnswererRewardFunction/std": 0.1789802610874176,
183
+ "step": 9,
184
+ "step_time": 8.014759111000103
185
+ },
186
+ {
187
+ "clip_ratio/high_max": 0.009765625,
188
+ "clip_ratio/high_mean": 0.009765625,
189
+ "clip_ratio/low_mean": 0.0048828125,
190
+ "clip_ratio/low_min": 0.0048828125,
191
+ "clip_ratio/region_mean": 0.0146484375,
192
+ "entropy": 0.743914008140564,
193
+ "epoch": 0.25,
194
+ "grad_norm": 2.893528699874878,
195
+ "kl": 0.08459636569023132,
196
+ "learning_rate": 2.775e-06,
197
+ "loss": -0.013386903330683708,
198
+ "step": 10,
199
+ "step_time": 0.11163025199994081
200
+ },
201
+ {
202
+ "clip_ratio/high_max": 0.0029296875,
203
+ "clip_ratio/high_mean": 0.0029296875,
204
+ "clip_ratio/low_mean": 0.021484375,
205
+ "clip_ratio/low_min": 0.021484375,
206
+ "clip_ratio/region_mean": 0.0244140625,
207
+ "entropy": 0.697109580039978,
208
+ "epoch": 0.275,
209
+ "grad_norm": 2.882779359817505,
210
+ "kl": 0.04983264207839966,
211
+ "learning_rate": 2.75e-06,
212
+ "loss": 0.006847226992249489,
213
+ "step": 11,
214
+ "step_time": 0.11213077200000043
215
+ },
216
+ {
217
+ "clip_ratio/high_max": 0.0,
218
+ "clip_ratio/high_mean": 0.0,
219
+ "clip_ratio/low_mean": 0.041015625,
220
+ "clip_ratio/low_min": 0.041015625,
221
+ "clip_ratio/region_mean": 0.041015625,
222
+ "entropy": 1.1653341054916382,
223
+ "epoch": 0.3,
224
+ "grad_norm": 3.08870267868042,
225
+ "kl": 0.08870606869459152,
226
+ "learning_rate": 2.725e-06,
227
+ "loss": 0.13892900943756104,
228
+ "step": 12,
229
+ "step_time": 0.11271204500008025
230
+ },
231
+ {
232
+ "clip_ratio/high_max": 0.0009765625,
233
+ "clip_ratio/high_mean": 0.0009765625,
234
+ "clip_ratio/low_mean": 0.0,
235
+ "clip_ratio/low_min": 0.0,
236
+ "clip_ratio/region_mean": 0.0009765625,
237
+ "completions/clipped_ratio": 1.0,
238
+ "completions/max_length": 256.0,
239
+ "completions/max_terminated_length": 0.0,
240
+ "completions/mean_length": 256.0,
241
+ "completions/mean_terminated_length": 0.0,
242
+ "completions/min_length": 256.0,
243
+ "completions/min_terminated_length": 0.0,
244
+ "entropy": 0.63991379737854,
245
+ "epoch": 0.325,
246
+ "frac_reward_zero_std": 0.0,
247
+ "grad_norm": 3.3573482036590576,
248
+ "kl": 0.06412748992443085,
249
+ "learning_rate": 2.7e-06,
250
+ "loss": -0.16908901929855347,
251
+ "num_tokens": 39416.0,
252
+ "reward": 0.6549176573753357,
253
+ "reward_std": 0.09264716506004333,
254
+ "rewards/AnswererRewardFunction/mean": 0.6549176573753357,
255
+ "rewards/AnswererRewardFunction/std": 0.09264715760946274,
256
+ "step": 13,
257
+ "step_time": 7.928827267000088
258
+ },
259
+ {
260
+ "clip_ratio/high_max": 0.001953125,
261
+ "clip_ratio/high_mean": 0.001953125,
262
+ "clip_ratio/low_mean": 0.001953125,
263
+ "clip_ratio/low_min": 0.001953125,
264
+ "clip_ratio/region_mean": 0.00390625,
265
+ "entropy": 0.3732947111129761,
266
+ "epoch": 0.35,
267
+ "grad_norm": 1.7340325117111206,
268
+ "kl": 0.043330118060112,
269
+ "learning_rate": 2.6750000000000002e-06,
270
+ "loss": 0.006798031274229288,
271
+ "step": 14,
272
+ "step_time": 0.11074772500001018
273
+ },
274
+ {
275
+ "clip_ratio/high_max": 0.01171875,
276
+ "clip_ratio/high_mean": 0.01171875,
277
+ "clip_ratio/low_mean": 0.009765625,
278
+ "clip_ratio/low_min": 0.009765625,
279
+ "clip_ratio/region_mean": 0.021484375,
280
+ "entropy": 0.8068607449531555,
281
+ "epoch": 0.375,
282
+ "grad_norm": 3.2802722454071045,
283
+ "kl": 0.0741429328918457,
284
+ "learning_rate": 2.65e-06,
285
+ "loss": 0.05089528113603592,
286
+ "step": 15,
287
+ "step_time": 0.11073857499991391
288
+ },
289
+ {
290
+ "clip_ratio/high_max": 0.015625,
291
+ "clip_ratio/high_mean": 0.015625,
292
+ "clip_ratio/low_mean": 0.0166015625,
293
+ "clip_ratio/low_min": 0.0166015625,
294
+ "clip_ratio/region_mean": 0.0322265625,
295
+ "entropy": 0.7192014455795288,
296
+ "epoch": 0.4,
297
+ "grad_norm": 4.314362049102783,
298
+ "kl": 0.07965575158596039,
299
+ "learning_rate": 2.6250000000000003e-06,
300
+ "loss": 0.11372166872024536,
301
+ "step": 16,
302
+ "step_time": 0.11177824599985797
303
+ },
304
+ {
305
+ "clip_ratio/high_max": 0.0,
306
+ "clip_ratio/high_mean": 0.0,
307
+ "clip_ratio/low_mean": 0.0,
308
+ "clip_ratio/low_min": 0.0,
309
+ "clip_ratio/region_mean": 0.0,
310
+ "completions/clipped_ratio": 1.0,
311
+ "completions/max_length": 256.0,
312
+ "completions/max_terminated_length": 0.0,
313
+ "completions/mean_length": 256.0,
314
+ "completions/mean_terminated_length": 0.0,
315
+ "completions/min_length": 256.0,
316
+ "completions/min_terminated_length": 0.0,
317
+ "entropy": 0.6362192630767822,
318
+ "epoch": 0.425,
319
+ "frac_reward_zero_std": 0.0,
320
+ "grad_norm": 3.0532214641571045,
321
+ "kl": 0.0684322863817215,
322
+ "learning_rate": 2.6e-06,
323
+ "loss": 0.0310845747590065,
324
+ "num_tokens": 49824.0,
325
+ "reward": 0.8395085334777832,
326
+ "reward_std": 0.15289129316806793,
327
+ "rewards/AnswererRewardFunction/mean": 0.8395085334777832,
328
+ "rewards/AnswererRewardFunction/std": 0.15289129316806793,
329
+ "step": 17,
330
+ "step_time": 7.96237816200005
331
+ },
332
+ {
333
+ "clip_ratio/high_max": 0.0068359375,
334
+ "clip_ratio/high_mean": 0.0068359375,
335
+ "clip_ratio/low_mean": 0.00390625,
336
+ "clip_ratio/low_min": 0.00390625,
337
+ "clip_ratio/region_mean": 0.0107421875,
338
+ "entropy": 0.6381477117538452,
339
+ "epoch": 0.45,
340
+ "grad_norm": 4.106065273284912,
341
+ "kl": 0.09328670799732208,
342
+ "learning_rate": 2.575e-06,
343
+ "loss": -0.07613930106163025,
344
+ "step": 18,
345
+ "step_time": 0.1118761339998855
346
+ },
347
+ {
348
+ "clip_ratio/high_max": 0.005859375,
349
+ "clip_ratio/high_mean": 0.005859375,
350
+ "clip_ratio/low_mean": 0.013671875,
351
+ "clip_ratio/low_min": 0.013671875,
352
+ "clip_ratio/region_mean": 0.01953125,
353
+ "entropy": 0.5689660310745239,
354
+ "epoch": 0.475,
355
+ "grad_norm": 1.5947890281677246,
356
+ "kl": 0.08299237489700317,
357
+ "learning_rate": 2.55e-06,
358
+ "loss": 0.03764766454696655,
359
+ "step": 19,
360
+ "step_time": 0.1142826679999871
361
+ },
362
+ {
363
+ "clip_ratio/high_max": 0.0126953125,
364
+ "clip_ratio/high_mean": 0.0126953125,
365
+ "clip_ratio/low_mean": 0.0107421875,
366
+ "clip_ratio/low_min": 0.0107421875,
367
+ "clip_ratio/region_mean": 0.0234375,
368
+ "entropy": 0.6226636171340942,
369
+ "epoch": 0.5,
370
+ "grad_norm": 1.9378103017807007,
371
+ "kl": 0.08963169902563095,
372
+ "learning_rate": 2.525e-06,
373
+ "loss": 0.008068917319178581,
374
+ "step": 20,
375
+ "step_time": 0.11152214599997023
376
+ },
377
+ {
378
+ "clip_ratio/high_max": 0.0,
379
+ "clip_ratio/high_mean": 0.0,
380
+ "clip_ratio/low_mean": 0.0,
381
+ "clip_ratio/low_min": 0.0,
382
+ "clip_ratio/region_mean": 0.0,
383
+ "completions/clipped_ratio": 1.0,
384
+ "completions/max_length": 256.0,
385
+ "completions/max_terminated_length": 0.0,
386
+ "completions/mean_length": 256.0,
387
+ "completions/mean_terminated_length": 0.0,
388
+ "completions/min_length": 256.0,
389
+ "completions/min_terminated_length": 0.0,
390
+ "entropy": 0.7389768362045288,
391
+ "epoch": 0.525,
392
+ "frac_reward_zero_std": 0.0,
393
+ "grad_norm": 3.434631109237671,
394
+ "kl": 0.12693551182746887,
395
+ "learning_rate": 2.5e-06,
396
+ "loss": -0.023861240595579147,
397
+ "num_tokens": 59924.0,
398
+ "reward": 0.6655067205429077,
399
+ "reward_std": 0.17209242284297943,
400
+ "rewards/AnswererRewardFunction/mean": 0.6655067205429077,
401
+ "rewards/AnswererRewardFunction/std": 0.17209243774414062,
402
+ "step": 21,
403
+ "step_time": 7.950470066999969
404
+ },
405
+ {
406
+ "clip_ratio/high_max": 0.0048828125,
407
+ "clip_ratio/high_mean": 0.0048828125,
408
+ "clip_ratio/low_mean": 0.0029296875,
409
+ "clip_ratio/low_min": 0.0029296875,
410
+ "clip_ratio/region_mean": 0.0078125,
411
+ "entropy": 0.6952782869338989,
412
+ "epoch": 0.55,
413
+ "grad_norm": 3.3466062545776367,
414
+ "kl": 0.09632558375597,
415
+ "learning_rate": 2.475e-06,
416
+ "loss": -0.015093192458152771,
417
+ "step": 22,
418
+ "step_time": 0.11464352500001951
419
+ },
420
+ {
421
+ "clip_ratio/high_max": 0.0078125,
422
+ "clip_ratio/high_mean": 0.0078125,
423
+ "clip_ratio/low_mean": 0.015625,
424
+ "clip_ratio/low_min": 0.015625,
425
+ "clip_ratio/region_mean": 0.0234375,
426
+ "entropy": 0.4058366119861603,
427
+ "epoch": 0.575,
428
+ "grad_norm": 1.916627287864685,
429
+ "kl": 0.10934790968894958,
430
+ "learning_rate": 2.45e-06,
431
+ "loss": 0.038687966763973236,
432
+ "step": 23,
433
+ "step_time": 0.11371357500001977
434
+ },
435
+ {
436
+ "clip_ratio/high_max": 0.0107421875,
437
+ "clip_ratio/high_mean": 0.0107421875,
438
+ "clip_ratio/low_mean": 0.005859375,
439
+ "clip_ratio/low_min": 0.005859375,
440
+ "clip_ratio/region_mean": 0.0166015625,
441
+ "entropy": 0.4684597849845886,
442
+ "epoch": 0.6,
443
+ "grad_norm": 2.56581449508667,
444
+ "kl": 0.09573253244161606,
445
+ "learning_rate": 2.425e-06,
446
+ "loss": 0.0021250469144433737,
447
+ "step": 24,
448
+ "step_time": 0.11208250999993652
449
+ },
450
+ {
451
+ "clip_ratio/high_max": 0.0,
452
+ "clip_ratio/high_mean": 0.0,
453
+ "clip_ratio/low_mean": 0.0,
454
+ "clip_ratio/low_min": 0.0,
455
+ "clip_ratio/region_mean": 0.0,
456
+ "completions/clipped_ratio": 1.0,
457
+ "completions/max_length": 256.0,
458
+ "completions/max_terminated_length": 0.0,
459
+ "completions/mean_length": 256.0,
460
+ "completions/mean_terminated_length": 0.0,
461
+ "completions/min_length": 256.0,
462
+ "completions/min_terminated_length": 0.0,
463
+ "entropy": 0.5364047288894653,
464
+ "epoch": 0.625,
465
+ "frac_reward_zero_std": 0.0,
466
+ "grad_norm": 3.1820530891418457,
467
+ "kl": 0.11052386462688446,
468
+ "learning_rate": 2.4000000000000003e-06,
469
+ "loss": -0.12420213222503662,
470
+ "num_tokens": 69996.0,
471
+ "reward": 0.706213653087616,
472
+ "reward_std": 0.22473102807998657,
473
+ "rewards/AnswererRewardFunction/mean": 0.706213653087616,
474
+ "rewards/AnswererRewardFunction/std": 0.22473104298114777,
475
+ "step": 25,
476
+ "step_time": 7.935186696000073
477
+ },
478
+ {
479
+ "clip_ratio/high_max": 0.0068359375,
480
+ "clip_ratio/high_mean": 0.0068359375,
481
+ "clip_ratio/low_mean": 0.0,
482
+ "clip_ratio/low_min": 0.0,
483
+ "clip_ratio/region_mean": 0.0068359375,
484
+ "entropy": 0.5579304695129395,
485
+ "epoch": 0.65,
486
+ "grad_norm": 3.2145562171936035,
487
+ "kl": 0.09837664663791656,
488
+ "learning_rate": 2.375e-06,
489
+ "loss": -0.048176076263189316,
490
+ "step": 26,
491
+ "step_time": 0.11106458200015368
492
+ },
493
+ {
494
+ "clip_ratio/high_max": 0.005859375,
495
+ "clip_ratio/high_mean": 0.005859375,
496
+ "clip_ratio/low_mean": 0.0048828125,
497
+ "clip_ratio/low_min": 0.0048828125,
498
+ "clip_ratio/region_mean": 0.0107421875,
499
+ "entropy": 0.5162898898124695,
500
+ "epoch": 0.675,
501
+ "grad_norm": 2.879662036895752,
502
+ "kl": 0.11825042963027954,
503
+ "learning_rate": 2.35e-06,
504
+ "loss": 0.06364448368549347,
505
+ "step": 27,
506
+ "step_time": 0.11082449600007749
507
+ },
508
+ {
509
+ "clip_ratio/high_max": 0.0048828125,
510
+ "clip_ratio/high_mean": 0.0048828125,
511
+ "clip_ratio/low_mean": 0.0068359375,
512
+ "clip_ratio/low_min": 0.0068359375,
513
+ "clip_ratio/region_mean": 0.01171875,
514
+ "entropy": 0.4608369469642639,
515
+ "epoch": 0.7,
516
+ "grad_norm": 2.5901477336883545,
517
+ "kl": 0.09662161022424698,
518
+ "learning_rate": 2.325e-06,
519
+ "loss": 0.11211410164833069,
520
+ "step": 28,
521
+ "step_time": 0.11086178800019297
522
+ },
523
+ {
524
+ "clip_ratio/high_max": 0.0,
525
+ "clip_ratio/high_mean": 0.0,
526
+ "clip_ratio/low_mean": 0.0,
527
+ "clip_ratio/low_min": 0.0,
528
+ "clip_ratio/region_mean": 0.0,
529
+ "completions/clipped_ratio": 1.0,
530
+ "completions/max_length": 256.0,
531
+ "completions/max_terminated_length": 0.0,
532
+ "completions/mean_length": 256.0,
533
+ "completions/mean_terminated_length": 0.0,
534
+ "completions/min_length": 256.0,
535
+ "completions/min_terminated_length": 0.0,
536
+ "entropy": 0.42433732748031616,
537
+ "epoch": 0.725,
538
+ "frac_reward_zero_std": 0.0,
539
+ "grad_norm": 2.176980495452881,
540
+ "kl": 0.10524210333824158,
541
+ "learning_rate": 2.3000000000000004e-06,
542
+ "loss": 0.008818887174129486,
543
+ "num_tokens": 80208.0,
544
+ "reward": 0.7267547845840454,
545
+ "reward_std": 0.179252490401268,
546
+ "rewards/AnswererRewardFunction/mean": 0.7267547845840454,
547
+ "rewards/AnswererRewardFunction/std": 0.1792525053024292,
548
+ "step": 29,
549
+ "step_time": 7.899287777999916
550
+ },
551
+ {
552
+ "clip_ratio/high_max": 0.0,
553
+ "clip_ratio/high_mean": 0.0,
554
+ "clip_ratio/low_mean": 0.005859375,
555
+ "clip_ratio/low_min": 0.005859375,
556
+ "clip_ratio/region_mean": 0.005859375,
557
+ "entropy": 0.7436986565589905,
558
+ "epoch": 0.75,
559
+ "grad_norm": 5.134580612182617,
560
+ "kl": 0.12144140899181366,
561
+ "learning_rate": 2.275e-06,
562
+ "loss": 0.20647373795509338,
563
+ "step": 30,
564
+ "step_time": 0.11205311000003348
565
+ },
566
+ {
567
+ "clip_ratio/high_max": 0.0068359375,
568
+ "clip_ratio/high_mean": 0.0068359375,
569
+ "clip_ratio/low_mean": 0.00390625,
570
+ "clip_ratio/low_min": 0.00390625,
571
+ "clip_ratio/region_mean": 0.0107421875,
572
+ "entropy": 0.5429961681365967,
573
+ "epoch": 0.775,
574
+ "grad_norm": 2.0209434032440186,
575
+ "kl": 0.12714454531669617,
576
+ "learning_rate": 2.25e-06,
577
+ "loss": -0.05760453641414642,
578
+ "step": 31,
579
+ "step_time": 0.11339481400000295
580
+ },
581
+ {
582
+ "clip_ratio/high_max": 0.01953125,
583
+ "clip_ratio/high_mean": 0.01953125,
584
+ "clip_ratio/low_mean": 0.0,
585
+ "clip_ratio/low_min": 0.0,
586
+ "clip_ratio/region_mean": 0.01953125,
587
+ "entropy": 0.32000452280044556,
588
+ "epoch": 0.8,
589
+ "grad_norm": 1.7520897388458252,
590
+ "kl": 0.11870498955249786,
591
+ "learning_rate": 2.2250000000000003e-06,
592
+ "loss": -0.1561477929353714,
593
+ "step": 32,
594
+ "step_time": 0.11099254700002348
595
+ },
596
+ {
597
+ "clip_ratio/high_max": 0.0,
598
+ "clip_ratio/high_mean": 0.0,
599
+ "clip_ratio/low_mean": 0.0,
600
+ "clip_ratio/low_min": 0.0,
601
+ "clip_ratio/region_mean": 0.0,
602
+ "completions/clipped_ratio": 1.0,
603
+ "completions/max_length": 256.0,
604
+ "completions/max_terminated_length": 0.0,
605
+ "completions/mean_length": 256.0,
606
+ "completions/mean_terminated_length": 0.0,
607
+ "completions/min_length": 256.0,
608
+ "completions/min_terminated_length": 0.0,
609
+ "entropy": 0.37099331617355347,
610
+ "epoch": 0.825,
611
+ "frac_reward_zero_std": 0.0,
612
+ "grad_norm": 2.467416763305664,
613
+ "kl": 0.11815842986106873,
614
+ "learning_rate": 2.1999999999999997e-06,
615
+ "loss": -0.02738424763083458,
616
+ "num_tokens": 90620.0,
617
+ "reward": 0.8727706074714661,
618
+ "reward_std": 0.1972254365682602,
619
+ "rewards/AnswererRewardFunction/mean": 0.8727706074714661,
620
+ "rewards/AnswererRewardFunction/std": 0.1972254365682602,
621
+ "step": 33,
622
+ "step_time": 7.91921779900008
623
+ },
624
+ {
625
+ "clip_ratio/high_max": 0.0009765625,
626
+ "clip_ratio/high_mean": 0.0009765625,
627
+ "clip_ratio/low_mean": 0.00390625,
628
+ "clip_ratio/low_min": 0.00390625,
629
+ "clip_ratio/region_mean": 0.0048828125,
630
+ "entropy": 0.3960208296775818,
631
+ "epoch": 0.85,
632
+ "grad_norm": 2.1480352878570557,
633
+ "kl": 0.09754370152950287,
634
+ "learning_rate": 2.175e-06,
635
+ "loss": 0.060052644461393356,
636
+ "step": 34,
637
+ "step_time": 0.11552422499994464
638
+ },
639
+ {
640
+ "clip_ratio/high_max": 0.0107421875,
641
+ "clip_ratio/high_mean": 0.0107421875,
642
+ "clip_ratio/low_mean": 0.0048828125,
643
+ "clip_ratio/low_min": 0.0048828125,
644
+ "clip_ratio/region_mean": 0.015625,
645
+ "entropy": 0.542590856552124,
646
+ "epoch": 0.875,
647
+ "grad_norm": 3.6754674911499023,
648
+ "kl": 0.10792969167232513,
649
+ "learning_rate": 2.15e-06,
650
+ "loss": -0.0720224380493164,
651
+ "step": 35,
652
+ "step_time": 0.11197718300013548
653
+ },
654
+ {
655
+ "clip_ratio/high_max": 0.013671875,
656
+ "clip_ratio/high_mean": 0.013671875,
657
+ "clip_ratio/low_mean": 0.0087890625,
658
+ "clip_ratio/low_min": 0.0087890625,
659
+ "clip_ratio/region_mean": 0.0224609375,
660
+ "entropy": 0.3774242401123047,
661
+ "epoch": 0.9,
662
+ "grad_norm": 1250253952.0,
663
+ "kl": 23265218.0,
664
+ "learning_rate": 2.125e-06,
665
+ "loss": 58163.08203125,
666
+ "step": 36,
667
+ "step_time": 0.12223055300000851
668
+ },
669
+ {
670
+ "clip_ratio/high_max": 0.0009765625,
671
+ "clip_ratio/high_mean": 0.0009765625,
672
+ "clip_ratio/low_mean": 0.0,
673
+ "clip_ratio/low_min": 0.0,
674
+ "clip_ratio/region_mean": 0.0009765625,
675
+ "completions/clipped_ratio": 1.0,
676
+ "completions/max_length": 256.0,
677
+ "completions/max_terminated_length": 0.0,
678
+ "completions/mean_length": 256.0,
679
+ "completions/mean_terminated_length": 0.0,
680
+ "completions/min_length": 256.0,
681
+ "completions/min_terminated_length": 0.0,
682
+ "entropy": 0.5863927602767944,
683
+ "epoch": 0.925,
684
+ "frac_reward_zero_std": 0.25,
685
+ "grad_norm": 1.7186689376831055,
686
+ "kl": 0.12184222787618637,
687
+ "learning_rate": 2.1e-06,
688
+ "loss": 0.01318025216460228,
689
+ "num_tokens": 100744.0,
690
+ "reward": 0.8322785496711731,
691
+ "reward_std": 0.17467200756072998,
692
+ "rewards/AnswererRewardFunction/mean": 0.8322785496711731,
693
+ "rewards/AnswererRewardFunction/std": 0.17467200756072998,
694
+ "step": 37,
695
+ "step_time": 7.957934481999928
696
+ },
697
+ {
698
+ "clip_ratio/high_max": 0.0078125,
699
+ "clip_ratio/high_mean": 0.0078125,
700
+ "clip_ratio/low_mean": 0.0,
701
+ "clip_ratio/low_min": 0.0,
702
+ "clip_ratio/region_mean": 0.0078125,
703
+ "entropy": 0.4571380019187927,
704
+ "epoch": 0.95,
705
+ "grad_norm": 2.5918664932250977,
706
+ "kl": 0.15713587403297424,
707
+ "learning_rate": 2.075e-06,
708
+ "loss": -0.15538358688354492,
709
+ "step": 38,
710
+ "step_time": 0.11134468399995967
711
+ },
712
+ {
713
+ "clip_ratio/high_max": 0.009765625,
714
+ "clip_ratio/high_mean": 0.009765625,
715
+ "clip_ratio/low_mean": 0.001953125,
716
+ "clip_ratio/low_min": 0.001953125,
717
+ "clip_ratio/region_mean": 0.01171875,
718
+ "entropy": 0.5838406085968018,
719
+ "epoch": 0.975,
720
+ "grad_norm": 3.389342784881592,
721
+ "kl": 0.16417548060417175,
722
+ "learning_rate": 2.0500000000000003e-06,
723
+ "loss": 0.036985911428928375,
724
+ "step": 39,
725
+ "step_time": 0.11151985699984834
726
+ },
727
+ {
728
+ "clip_ratio/high_max": 0.0068359375,
729
+ "clip_ratio/high_mean": 0.0068359375,
730
+ "clip_ratio/low_mean": 0.005859375,
731
+ "clip_ratio/low_min": 0.005859375,
732
+ "clip_ratio/region_mean": 0.0126953125,
733
+ "entropy": 0.626500129699707,
734
+ "epoch": 1.0,
735
+ "grad_norm": 3.443002939224243,
736
+ "kl": 0.19608259201049805,
737
+ "learning_rate": 2.025e-06,
738
+ "loss": 0.1089545264840126,
739
+ "step": 40,
740
+ "step_time": 0.11399926900003265
741
+ },
742
+ {
743
+ "clip_ratio/high_max": 0.0,
744
+ "clip_ratio/high_mean": 0.0,
745
+ "clip_ratio/low_mean": 0.0009765625,
746
+ "clip_ratio/low_min": 0.0009765625,
747
+ "clip_ratio/region_mean": 0.0009765625,
748
+ "completions/clipped_ratio": 1.0,
749
+ "completions/max_length": 256.0,
750
+ "completions/max_terminated_length": 0.0,
751
+ "completions/mean_length": 256.0,
752
+ "completions/mean_terminated_length": 0.0,
753
+ "completions/min_length": 256.0,
754
+ "completions/min_terminated_length": 0.0,
755
+ "entropy": 0.5399507880210876,
756
+ "epoch": 1.025,
757
+ "frac_reward_zero_std": 0.0,
758
+ "grad_norm": 2.0896756649017334,
759
+ "kl": 0.16342155635356903,
760
+ "learning_rate": 2e-06,
761
+ "loss": 0.09698553383350372,
762
+ "num_tokens": 110360.0,
763
+ "reward": 0.6938474774360657,
764
+ "reward_std": 0.07210177928209305,
765
+ "rewards/AnswererRewardFunction/mean": 0.6938474774360657,
766
+ "rewards/AnswererRewardFunction/std": 0.07210177928209305,
767
+ "step": 41,
768
+ "step_time": 8.036780455000098
769
+ },
770
+ {
771
+ "clip_ratio/high_max": 0.0048828125,
772
+ "clip_ratio/high_mean": 0.0048828125,
773
+ "clip_ratio/low_mean": 0.001953125,
774
+ "clip_ratio/low_min": 0.001953125,
775
+ "clip_ratio/region_mean": 0.0068359375,
776
+ "entropy": 0.5720961689949036,
777
+ "epoch": 1.05,
778
+ "grad_norm": 3.426593065261841,
779
+ "kl": 0.19985011219978333,
780
+ "learning_rate": 1.975e-06,
781
+ "loss": -0.014200590550899506,
782
+ "step": 42,
783
+ "step_time": 0.11001263299999664
784
+ },
785
+ {
786
+ "clip_ratio/high_max": 0.009765625,
787
+ "clip_ratio/high_mean": 0.009765625,
788
+ "clip_ratio/low_mean": 0.0048828125,
789
+ "clip_ratio/low_min": 0.0048828125,
790
+ "clip_ratio/region_mean": 0.0146484375,
791
+ "entropy": 0.5380070209503174,
792
+ "epoch": 1.075,
793
+ "grad_norm": 1.9040859937667847,
794
+ "kl": 0.1798008680343628,
795
+ "learning_rate": 1.95e-06,
796
+ "loss": -0.0504855252802372,
797
+ "step": 43,
798
+ "step_time": 0.10936977100004697
799
+ },
800
+ {
801
+ "clip_ratio/high_max": 0.01953125,
802
+ "clip_ratio/high_mean": 0.01953125,
803
+ "clip_ratio/low_mean": 0.0029296875,
804
+ "clip_ratio/low_min": 0.0029296875,
805
+ "clip_ratio/region_mean": 0.0224609375,
806
+ "entropy": 0.45030832290649414,
807
+ "epoch": 1.1,
808
+ "grad_norm": 2.7696876525878906,
809
+ "kl": 0.1864057183265686,
810
+ "learning_rate": 1.925e-06,
811
+ "loss": -0.029610613361001015,
812
+ "step": 44,
813
+ "step_time": 0.11253336699996908
814
+ },
815
+ {
816
+ "clip_ratio/high_max": 0.0,
817
+ "clip_ratio/high_mean": 0.0,
818
+ "clip_ratio/low_mean": 0.0,
819
+ "clip_ratio/low_min": 0.0,
820
+ "clip_ratio/region_mean": 0.0,
821
+ "completions/clipped_ratio": 1.0,
822
+ "completions/max_length": 256.0,
823
+ "completions/max_terminated_length": 0.0,
824
+ "completions/mean_length": 256.0,
825
+ "completions/mean_terminated_length": 0.0,
826
+ "completions/min_length": 256.0,
827
+ "completions/min_terminated_length": 0.0,
828
+ "entropy": 0.48534196615219116,
829
+ "epoch": 1.125,
830
+ "frac_reward_zero_std": 0.0,
831
+ "grad_norm": 2.1930065155029297,
832
+ "kl": 0.21630029380321503,
833
+ "learning_rate": 1.9e-06,
834
+ "loss": -0.0531463548541069,
835
+ "num_tokens": 120316.0,
836
+ "reward": 0.8261469602584839,
837
+ "reward_std": 0.13625741004943848,
838
+ "rewards/AnswererRewardFunction/mean": 0.8261469602584839,
839
+ "rewards/AnswererRewardFunction/std": 0.13625742495059967,
840
+ "step": 45,
841
+ "step_time": 7.959708546999991
842
+ },
843
+ {
844
+ "clip_ratio/high_max": 0.001953125,
845
+ "clip_ratio/high_mean": 0.001953125,
846
+ "clip_ratio/low_mean": 0.0009765625,
847
+ "clip_ratio/low_min": 0.0009765625,
848
+ "clip_ratio/region_mean": 0.0029296875,
849
+ "entropy": 0.5528072714805603,
850
+ "epoch": 1.15,
851
+ "grad_norm": 3.234687328338623,
852
+ "kl": 0.20044510066509247,
853
+ "learning_rate": 1.875e-06,
854
+ "loss": -0.14086312055587769,
855
+ "step": 46,
856
+ "step_time": 0.11803400899998451
857
+ },
858
+ {
859
+ "clip_ratio/high_max": 0.0068359375,
860
+ "clip_ratio/high_mean": 0.0068359375,
861
+ "clip_ratio/low_mean": 0.0068359375,
862
+ "clip_ratio/low_min": 0.0068359375,
863
+ "clip_ratio/region_mean": 0.013671875,
864
+ "entropy": 0.5274936556816101,
865
+ "epoch": 1.175,
866
+ "grad_norm": 2.5730295181274414,
867
+ "kl": 0.16597405076026917,
868
+ "learning_rate": 1.85e-06,
869
+ "loss": 0.023598745465278625,
870
+ "step": 47,
871
+ "step_time": 0.12857143599990195
872
+ },
873
+ {
874
+ "clip_ratio/high_max": 0.0,
875
+ "clip_ratio/high_mean": 0.0,
876
+ "clip_ratio/low_mean": 0.0166015625,
877
+ "clip_ratio/low_min": 0.0166015625,
878
+ "clip_ratio/region_mean": 0.0166015625,
879
+ "entropy": 0.5713120102882385,
880
+ "epoch": 1.2,
881
+ "grad_norm": 3.3215487003326416,
882
+ "kl": 0.15826421976089478,
883
+ "learning_rate": 1.8249999999999999e-06,
884
+ "loss": 0.1730981469154358,
885
+ "step": 48,
886
+ "step_time": 0.1108278810002048
887
+ },
888
+ {
889
+ "clip_ratio/high_max": 0.0009765625,
890
+ "clip_ratio/high_mean": 0.0009765625,
891
+ "clip_ratio/low_mean": 0.0009765625,
892
+ "clip_ratio/low_min": 0.0009765625,
893
+ "clip_ratio/region_mean": 0.001953125,
894
+ "completions/clipped_ratio": 1.0,
895
+ "completions/max_length": 256.0,
896
+ "completions/max_terminated_length": 0.0,
897
+ "completions/mean_length": 256.0,
898
+ "completions/mean_terminated_length": 0.0,
899
+ "completions/min_length": 256.0,
900
+ "completions/min_terminated_length": 0.0,
901
+ "entropy": 0.40691593289375305,
902
+ "epoch": 1.225,
903
+ "frac_reward_zero_std": 0.0,
904
+ "grad_norm": 2.9191150665283203,
905
+ "kl": 0.16014470160007477,
906
+ "learning_rate": 1.8e-06,
907
+ "loss": 0.16662299633026123,
908
+ "num_tokens": 130272.0,
909
+ "reward": 0.8778884410858154,
910
+ "reward_std": 0.17971795797348022,
911
+ "rewards/AnswererRewardFunction/mean": 0.8778884410858154,
912
+ "rewards/AnswererRewardFunction/std": 0.17971795797348022,
913
+ "step": 49,
914
+ "step_time": 8.018302381000012
915
+ },
916
+ {
917
+ "clip_ratio/high_max": 0.0,
918
+ "clip_ratio/high_mean": 0.0,
919
+ "clip_ratio/low_mean": 0.00390625,
920
+ "clip_ratio/low_min": 0.00390625,
921
+ "clip_ratio/region_mean": 0.00390625,
922
+ "entropy": 0.3267471194267273,
923
+ "epoch": 1.25,
924
+ "grad_norm": 1.9587223529815674,
925
+ "kl": 0.17650750279426575,
926
+ "learning_rate": 1.7750000000000002e-06,
927
+ "loss": -0.019061673432588577,
928
+ "step": 50,
929
+ "step_time": 0.11307515899989085
930
+ },
931
+ {
932
+ "clip_ratio/high_max": 0.009765625,
933
+ "clip_ratio/high_mean": 0.009765625,
934
+ "clip_ratio/low_mean": 0.0068359375,
935
+ "clip_ratio/low_min": 0.0068359375,
936
+ "clip_ratio/region_mean": 0.0166015625,
937
+ "entropy": 0.47037309408187866,
938
+ "epoch": 1.275,
939
+ "grad_norm": 1.7802549600601196,
940
+ "kl": 0.15526388585567474,
941
+ "learning_rate": 1.7500000000000002e-06,
942
+ "loss": -0.04963531345129013,
943
+ "step": 51,
944
+ "step_time": 0.11180471299985584
945
+ },
946
+ {
947
+ "clip_ratio/high_max": 0.0068359375,
948
+ "clip_ratio/high_mean": 0.0068359375,
949
+ "clip_ratio/low_mean": 0.013671875,
950
+ "clip_ratio/low_min": 0.013671875,
951
+ "clip_ratio/region_mean": 0.0205078125,
952
+ "entropy": 0.5774157047271729,
953
+ "epoch": 1.3,
954
+ "grad_norm": 2.833186388015747,
955
+ "kl": 0.1678266078233719,
956
+ "learning_rate": 1.725e-06,
957
+ "loss": -0.093873992562294,
958
+ "step": 52,
959
+ "step_time": 0.112085678999847
960
+ },
961
+ {
962
+ "clip_ratio/high_max": 0.0,
963
+ "clip_ratio/high_mean": 0.0,
964
+ "clip_ratio/low_mean": 0.0,
965
+ "clip_ratio/low_min": 0.0,
966
+ "clip_ratio/region_mean": 0.0,
967
+ "completions/clipped_ratio": 1.0,
968
+ "completions/max_length": 256.0,
969
+ "completions/max_terminated_length": 0.0,
970
+ "completions/mean_length": 256.0,
971
+ "completions/mean_terminated_length": 0.0,
972
+ "completions/min_length": 256.0,
973
+ "completions/min_terminated_length": 0.0,
974
+ "entropy": 0.5357224941253662,
975
+ "epoch": 1.325,
976
+ "frac_reward_zero_std": 0.0,
977
+ "grad_norm": 3.081116199493408,
978
+ "kl": 0.18712827563285828,
979
+ "learning_rate": 1.7e-06,
980
+ "loss": 0.10657118260860443,
981
+ "num_tokens": 140436.0,
982
+ "reward": 0.7755283117294312,
983
+ "reward_std": 0.15775497257709503,
984
+ "rewards/AnswererRewardFunction/mean": 0.7755283117294312,
985
+ "rewards/AnswererRewardFunction/std": 0.15775495767593384,
986
+ "step": 53,
987
+ "step_time": 8.014508306999915
988
+ },
989
+ {
990
+ "clip_ratio/high_max": 0.0029296875,
991
+ "clip_ratio/high_mean": 0.0029296875,
992
+ "clip_ratio/low_mean": 0.005859375,
993
+ "clip_ratio/low_min": 0.005859375,
994
+ "clip_ratio/region_mean": 0.0087890625,
995
+ "entropy": 0.4421442747116089,
996
+ "epoch": 1.35,
997
+ "grad_norm": 2.3898942470550537,
998
+ "kl": 0.1956406980752945,
999
+ "learning_rate": 1.675e-06,
1000
+ "loss": 0.011319718323647976,
1001
+ "step": 54,
1002
+ "step_time": 0.11459642999989228
1003
+ },
1004
+ {
1005
+ "clip_ratio/high_max": 0.001953125,
1006
+ "clip_ratio/high_mean": 0.001953125,
1007
+ "clip_ratio/low_mean": 0.009765625,
1008
+ "clip_ratio/low_min": 0.009765625,
1009
+ "clip_ratio/region_mean": 0.01171875,
1010
+ "entropy": 0.4516857862472534,
1011
+ "epoch": 1.375,
1012
+ "grad_norm": 1.689841389656067,
1013
+ "kl": 0.18309825658798218,
1014
+ "learning_rate": 1.65e-06,
1015
+ "loss": 0.06909539550542831,
1016
+ "step": 55,
1017
+ "step_time": 0.11464596999985588
1018
+ },
1019
+ {
1020
+ "clip_ratio/high_max": 0.009765625,
1021
+ "clip_ratio/high_mean": 0.009765625,
1022
+ "clip_ratio/low_mean": 0.0,
1023
+ "clip_ratio/low_min": 0.0,
1024
+ "clip_ratio/region_mean": 0.009765625,
1025
+ "entropy": 0.3544921875,
1026
+ "epoch": 1.4,
1027
+ "grad_norm": 2.365453004837036,
1028
+ "kl": 0.16238607466220856,
1029
+ "learning_rate": 1.625e-06,
1030
+ "loss": -0.18329277634620667,
1031
+ "step": 56,
1032
+ "step_time": 0.1123857229999885
1033
+ },
1034
+ {
1035
+ "clip_ratio/high_max": 0.0,
1036
+ "clip_ratio/high_mean": 0.0,
1037
+ "clip_ratio/low_mean": 0.0009765625,
1038
+ "clip_ratio/low_min": 0.0009765625,
1039
+ "clip_ratio/region_mean": 0.0009765625,
1040
+ "completions/clipped_ratio": 1.0,
1041
+ "completions/max_length": 256.0,
1042
+ "completions/max_terminated_length": 0.0,
1043
+ "completions/mean_length": 256.0,
1044
+ "completions/mean_terminated_length": 0.0,
1045
+ "completions/min_length": 256.0,
1046
+ "completions/min_terminated_length": 0.0,
1047
+ "entropy": 0.6127996444702148,
1048
+ "epoch": 1.425,
1049
+ "frac_reward_zero_std": 0.0,
1050
+ "grad_norm": 4.42158842086792,
1051
+ "kl": 0.12734977900981903,
1052
+ "learning_rate": 1.6e-06,
1053
+ "loss": 0.23196423053741455,
1054
+ "num_tokens": 150864.0,
1055
+ "reward": 0.8476194143295288,
1056
+ "reward_std": 0.16014426946640015,
1057
+ "rewards/AnswererRewardFunction/mean": 0.8476194143295288,
1058
+ "rewards/AnswererRewardFunction/std": 0.16014428436756134,
1059
+ "step": 57,
1060
+ "step_time": 7.9517445380001845
1061
+ },
1062
+ {
1063
+ "clip_ratio/high_max": 0.00390625,
1064
+ "clip_ratio/high_mean": 0.00390625,
1065
+ "clip_ratio/low_mean": 0.001953125,
1066
+ "clip_ratio/low_min": 0.001953125,
1067
+ "clip_ratio/region_mean": 0.005859375,
1068
+ "entropy": 0.4672216773033142,
1069
+ "epoch": 1.45,
1070
+ "grad_norm": 2.5147714614868164,
1071
+ "kl": 0.13779059052467346,
1072
+ "learning_rate": 1.5750000000000002e-06,
1073
+ "loss": -0.06359703838825226,
1074
+ "step": 58,
1075
+ "step_time": 0.11299467099979665
1076
+ },
1077
+ {
1078
+ "clip_ratio/high_max": 0.0068359375,
1079
+ "clip_ratio/high_mean": 0.0068359375,
1080
+ "clip_ratio/low_mean": 0.0087890625,
1081
+ "clip_ratio/low_min": 0.0087890625,
1082
+ "clip_ratio/region_mean": 0.015625,
1083
+ "entropy": 0.48225951194763184,
1084
+ "epoch": 1.475,
1085
+ "grad_norm": 2.714545249938965,
1086
+ "kl": 0.1360112726688385,
1087
+ "learning_rate": 1.5500000000000002e-06,
1088
+ "loss": -0.08431179821491241,
1089
+ "step": 59,
1090
+ "step_time": 0.11132271799988303
1091
+ },
1092
+ {
1093
+ "clip_ratio/high_max": 0.001953125,
1094
+ "clip_ratio/high_mean": 0.001953125,
1095
+ "clip_ratio/low_mean": 0.009765625,
1096
+ "clip_ratio/low_min": 0.009765625,
1097
+ "clip_ratio/region_mean": 0.01171875,
1098
+ "entropy": 0.5878890752792358,
1099
+ "epoch": 1.5,
1100
+ "grad_norm": 2.2165496349334717,
1101
+ "kl": 0.1507602035999298,
1102
+ "learning_rate": 1.525e-06,
1103
+ "loss": -0.08282318711280823,
1104
+ "step": 60,
1105
+ "step_time": 0.11144096099997114
1106
+ }
1107
+ ],
1108
+ "logging_steps": 1,
1109
+ "max_steps": 120,
1110
+ "num_input_tokens_seen": 150864,
1111
+ "num_train_epochs": 3,
1112
+ "save_steps": 30,
1113
+ "stateful_callbacks": {
1114
+ "TrainerControl": {
1115
+ "args": {
1116
+ "should_epoch_stop": false,
1117
+ "should_evaluate": false,
1118
+ "should_log": false,
1119
+ "should_save": true,
1120
+ "should_training_stop": false
1121
+ },
1122
+ "attributes": {}
1123
+ }
1124
+ },
1125
+ "total_flos": 0.0,
1126
+ "train_batch_size": 4,
1127
+ "trial_name": null,
1128
+ "trial_params": null
1129
+ }
self_play_hf_l40s_full/round_001/answerer_train/checkpoint-60/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9aa1741e484209ae738c1b939926d2de381bea548a0b79516d821f11e36862f
3
+ size 7249