Hugging Face's logo

Siddeshwar1625
/
osint-checkpoints

Safetensors
Model card Files
xet
 Community
Siddeshwar1625 commited on
Commit
15f72a4
·
verified ·
1 Parent(s): 52231e9

Intermediate checkpoint upload step=90 (generator_train)

Browse files
Files changed (12) hide show
  1. .gitattributes +1 -0
  2. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/chat_template.jinja +54 -0
  3. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/config.json +57 -0
  4. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/generation_config.json +13 -0
  5. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/model.safetensors +3 -0
  6. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/optimizer.pt +3 -0
  7. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/rng_state.pth +3 -0
  8. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/scheduler.pt +3 -0
  9. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/tokenizer.json +3 -0
  10. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/tokenizer_config.json +32 -0
  11. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/trainer_state.json +1683 -0
  12. self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/training_args.bin +3 -0
.gitattributes CHANGED
@@ -64,3 +64,4 @@ self_play_hf_l40s_full/round_002/answerer_train/checkpoint-120/tokenizer.json fi
64
  self_play_hf_l40s_full/round_002/answerer_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  self_play_hf_l40s_full/round_003/generator_train/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
66
  self_play_hf_l40s_full/round_003/generator_train/checkpoint-60/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
64
  self_play_hf_l40s_full/round_002/answerer_train/final_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  self_play_hf_l40s_full/round_003/generator_train/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
66
  self_play_hf_l40s_full/round_003/generator_train/checkpoint-60/tokenizer.json filter=lfs diff=lfs merge=lfs -text
67
+ self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/tokenizer.json filter=lfs diff=lfs merge=lfs -text
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": null,
7
+ "dtype": "float32",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 896,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4864,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention"
38
+ ],
39
+ "max_position_embeddings": 32768,
40
+ "max_window_layers": 21,
41
+ "model_type": "qwen2",
42
+ "num_attention_heads": 14,
43
+ "num_hidden_layers": 24,
44
+ "num_key_value_heads": 2,
45
+ "pad_token_id": 151643,
46
+ "rms_norm_eps": 1e-06,
47
+ "rope_parameters": {
48
+ "rope_theta": 1000000.0,
49
+ "rope_type": "default"
50
+ },
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.6.2",
54
+ "use_cache": false,
55
+ "use_sliding_window": false,
56
+ "vocab_size": 151936
57
+ }
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "repetition_penalty": 1.1,
9
+ "temperature": 0.7,
10
+ "top_k": 20,
11
+ "top_p": 0.8,
12
+ "transformers_version": "5.6.2"
13
+ }
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29ef3d681e7d1dc4f8dff38b47ec48aa0a0211b050af29918f0a2aca97e61a73
3
+ size 1976163472
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba352aaf0626a46e4b197264e4cc09c5105021d9db9283953bce1ed8f5303303
3
+ size 3952509771
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5001a1413e890182ec459c1edbf745017684c89f471fbb0cafcaadbc1070a9f
3
+ size 14645
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9466ecaa07ea9f7aa3b70a452615802059d7a87e6220a892a7f18c7703f928c0
3
+ size 1465
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
3
+ size 11421892
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": true,
24
+ "local_files_only": false,
25
+ "model_max_length": 131072,
26
+ "pad_token": "<|endoftext|>",
27
+ "padding_side": "left",
28
+ "split_special_tokens": false,
29
+ "tokenizer_class": "Qwen2Tokenizer",
30
+ "truncation_side": "left",
31
+ "unk_token": null
32
+ }
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/trainer_state.json ADDED
@@ -0,0 +1,1683 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 3.75,
6
+ "eval_steps": 500,
7
+ "global_step": 90,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 1.0,
19
+ "completions/max_length": 384.0,
20
+ "completions/max_terminated_length": 0.0,
21
+ "completions/mean_length": 384.0,
22
+ "completions/mean_terminated_length": 0.0,
23
+ "completions/min_length": 384.0,
24
+ "completions/min_terminated_length": 0.0,
25
+ "entropy": 3.465953826904297,
26
+ "epoch": 0.041666666666666664,
27
+ "frac_reward_zero_std": 0.25,
28
+ "grad_norm": 2.099428176879883,
29
+ "kl": 0.0004248070763424039,
30
+ "learning_rate": 5e-06,
31
+ "loss": 0.060097068548202515,
32
+ "num_tokens": 25528.0,
33
+ "reward": -0.07093749940395355,
34
+ "reward_std": 0.07468196004629135,
35
+ "rewards/GeneratorRewardFunction/mean": -0.07093749940395355,
36
+ "rewards/GeneratorRewardFunction/std": 0.07468196004629135,
37
+ "step": 1,
38
+ "step_time": 12.34978560799982
39
+ },
40
+ {
41
+ "clip_ratio/high_max": 0.0325520820915699,
42
+ "clip_ratio/high_mean": 0.0325520820915699,
43
+ "clip_ratio/low_mean": 0.0,
44
+ "clip_ratio/low_min": 0.0,
45
+ "clip_ratio/region_mean": 0.0325520820915699,
46
+ "entropy": 3.2590296268463135,
47
+ "epoch": 0.08333333333333333,
48
+ "grad_norm": 1.8288654088974,
49
+ "kl": 0.0101504847407341,
50
+ "learning_rate": 4.958333333333334e-06,
51
+ "loss": -0.12309196591377258,
52
+ "step": 2,
53
+ "step_time": 0.22763466400010657
54
+ },
55
+ {
56
+ "clip_ratio/high_max": 0.0462239570915699,
57
+ "clip_ratio/high_mean": 0.0462239570915699,
58
+ "clip_ratio/low_mean": 0.025390625,
59
+ "clip_ratio/low_min": 0.025390625,
60
+ "clip_ratio/region_mean": 0.0716145858168602,
61
+ "entropy": 3.6230857372283936,
62
+ "epoch": 0.125,
63
+ "grad_norm": 3.8438098430633545,
64
+ "kl": 0.017077775672078133,
65
+ "learning_rate": 4.9166666666666665e-06,
66
+ "loss": 0.1317509412765503,
67
+ "step": 3,
68
+ "step_time": 0.22691247900002054
69
+ },
70
+ {
71
+ "clip_ratio/high_max": 0.0611979179084301,
72
+ "clip_ratio/high_mean": 0.0611979179084301,
73
+ "clip_ratio/low_mean": 0.0,
74
+ "clip_ratio/low_min": 0.0,
75
+ "clip_ratio/region_mean": 0.0611979179084301,
76
+ "entropy": 3.5926520824432373,
77
+ "epoch": 0.16666666666666666,
78
+ "grad_norm": 0.9084944128990173,
79
+ "kl": 0.07373055815696716,
80
+ "learning_rate": 4.875e-06,
81
+ "loss": -0.059622813016176224,
82
+ "step": 4,
83
+ "step_time": 0.22661109199998464
84
+ },
85
+ {
86
+ "clip_ratio/high_max": 0.0,
87
+ "clip_ratio/high_mean": 0.0,
88
+ "clip_ratio/low_mean": 0.0,
89
+ "clip_ratio/low_min": 0.0,
90
+ "clip_ratio/region_mean": 0.0,
91
+ "completions/clipped_ratio": 1.0,
92
+ "completions/max_length": 384.0,
93
+ "completions/max_terminated_length": 0.0,
94
+ "completions/mean_length": 384.0,
95
+ "completions/mean_terminated_length": 0.0,
96
+ "completions/min_length": 384.0,
97
+ "completions/min_terminated_length": 0.0,
98
+ "entropy": 3.3955090045928955,
99
+ "epoch": 0.20833333333333334,
100
+ "frac_reward_zero_std": 0.5,
101
+ "grad_norm": 2.3290393352508545,
102
+ "kl": 0.01860710047185421,
103
+ "learning_rate": 4.833333333333333e-06,
104
+ "loss": 0.031048545613884926,
105
+ "num_tokens": 50668.0,
106
+ "reward": -0.053437501192092896,
107
+ "reward_std": 0.008508573286235332,
108
+ "rewards/GeneratorRewardFunction/mean": -0.053437501192092896,
109
+ "rewards/GeneratorRewardFunction/std": 0.008508575148880482,
110
+ "step": 5,
111
+ "step_time": 12.336290615000053
112
+ },
113
+ {
114
+ "clip_ratio/high_max": 0.0026041667442768812,
115
+ "clip_ratio/high_mean": 0.0026041667442768812,
116
+ "clip_ratio/low_mean": 0.0,
117
+ "clip_ratio/low_min": 0.0,
118
+ "clip_ratio/region_mean": 0.0026041667442768812,
119
+ "entropy": 3.5622146129608154,
120
+ "epoch": 0.25,
121
+ "grad_norm": 0.841543436050415,
122
+ "kl": 0.030347585678100586,
123
+ "learning_rate": 4.791666666666668e-06,
124
+ "loss": -0.03095925785601139,
125
+ "step": 6,
126
+ "step_time": 0.2276808209999217
127
+ },
128
+ {
129
+ "clip_ratio/high_max": 0.0325520820915699,
130
+ "clip_ratio/high_mean": 0.0325520820915699,
131
+ "clip_ratio/low_mean": 0.0065104165114462376,
132
+ "clip_ratio/low_min": 0.0065104165114462376,
133
+ "clip_ratio/region_mean": 0.0390625,
134
+ "entropy": 3.7959840297698975,
135
+ "epoch": 0.2916666666666667,
136
+ "grad_norm": 2.492034435272217,
137
+ "kl": 0.03457685932517052,
138
+ "learning_rate": 4.75e-06,
139
+ "loss": 0.040058210492134094,
140
+ "step": 7,
141
+ "step_time": 0.22600915599969085
142
+ },
143
+ {
144
+ "clip_ratio/high_max": 0.03059895895421505,
145
+ "clip_ratio/high_mean": 0.03059895895421505,
146
+ "clip_ratio/low_mean": 0.0,
147
+ "clip_ratio/low_min": 0.0,
148
+ "clip_ratio/region_mean": 0.03059895895421505,
149
+ "entropy": 3.540424346923828,
150
+ "epoch": 0.3333333333333333,
151
+ "grad_norm": 0.8438955545425415,
152
+ "kl": 0.04309028387069702,
153
+ "learning_rate": 4.708333333333334e-06,
154
+ "loss": -0.03881740942597389,
155
+ "step": 8,
156
+ "step_time": 0.22509408699988853
157
+ },
158
+ {
159
+ "clip_ratio/high_max": 0.0,
160
+ "clip_ratio/high_mean": 0.0,
161
+ "clip_ratio/low_mean": 0.0,
162
+ "clip_ratio/low_min": 0.0,
163
+ "clip_ratio/region_mean": 0.0,
164
+ "completions/clipped_ratio": 1.0,
165
+ "completions/max_length": 384.0,
166
+ "completions/max_terminated_length": 0.0,
167
+ "completions/mean_length": 384.0,
168
+ "completions/mean_terminated_length": 0.0,
169
+ "completions/min_length": 384.0,
170
+ "completions/min_terminated_length": 0.0,
171
+ "entropy": 3.7777175903320312,
172
+ "epoch": 0.375,
173
+ "frac_reward_zero_std": 0.0,
174
+ "grad_norm": 3.2919111251831055,
175
+ "kl": 0.05869259312748909,
176
+ "learning_rate": 4.666666666666667e-06,
177
+ "loss": 0.12271104007959366,
178
+ "num_tokens": 75660.0,
179
+ "reward": -0.09218750894069672,
180
+ "reward_std": 0.14918632805347443,
181
+ "rewards/GeneratorRewardFunction/mean": -0.09218750894069672,
182
+ "rewards/GeneratorRewardFunction/std": 0.14918632805347443,
183
+ "step": 9,
184
+ "step_time": 12.324919436000073
185
+ },
186
+ {
187
+ "clip_ratio/high_max": 0.008463541977107525,
188
+ "clip_ratio/high_mean": 0.008463541977107525,
189
+ "clip_ratio/low_mean": 0.0013020833721384406,
190
+ "clip_ratio/low_min": 0.0013020833721384406,
191
+ "clip_ratio/region_mean": 0.009765625,
192
+ "entropy": 3.8905904293060303,
193
+ "epoch": 0.4166666666666667,
194
+ "grad_norm": 2.505467653274536,
195
+ "kl": 0.06301017850637436,
196
+ "learning_rate": 4.625000000000001e-06,
197
+ "loss": 0.003014707937836647,
198
+ "step": 10,
199
+ "step_time": 0.22274375999995755
200
+ },
201
+ {
202
+ "clip_ratio/high_max": 0.0338541679084301,
203
+ "clip_ratio/high_mean": 0.0338541679084301,
204
+ "clip_ratio/low_mean": 0.0,
205
+ "clip_ratio/low_min": 0.0,
206
+ "clip_ratio/region_mean": 0.0338541679084301,
207
+ "entropy": 4.123391628265381,
208
+ "epoch": 0.4583333333333333,
209
+ "grad_norm": 1.8695275783538818,
210
+ "kl": 0.0821770653128624,
211
+ "learning_rate": 4.583333333333333e-06,
212
+ "loss": -0.12824173271656036,
213
+ "step": 11,
214
+ "step_time": 0.2223108019998108
215
+ },
216
+ {
217
+ "clip_ratio/high_max": 0.0631510391831398,
218
+ "clip_ratio/high_mean": 0.0631510391831398,
219
+ "clip_ratio/low_mean": 0.010416666977107525,
220
+ "clip_ratio/low_min": 0.010416666977107525,
221
+ "clip_ratio/region_mean": 0.0735677108168602,
222
+ "entropy": 4.174784183502197,
223
+ "epoch": 0.5,
224
+ "grad_norm": 2.2471466064453125,
225
+ "kl": 0.07816394418478012,
226
+ "learning_rate": 4.541666666666667e-06,
227
+ "loss": 0.00318560260348022,
228
+ "step": 12,
229
+ "step_time": 0.22159122599987313
230
+ },
231
+ {
232
+ "clip_ratio/high_max": 0.0,
233
+ "clip_ratio/high_mean": 0.0,
234
+ "clip_ratio/low_mean": 0.0,
235
+ "clip_ratio/low_min": 0.0,
236
+ "clip_ratio/region_mean": 0.0,
237
+ "completions/clipped_ratio": 1.0,
238
+ "completions/max_length": 384.0,
239
+ "completions/max_terminated_length": 0.0,
240
+ "completions/mean_length": 384.0,
241
+ "completions/mean_terminated_length": 0.0,
242
+ "completions/min_length": 384.0,
243
+ "completions/min_terminated_length": 0.0,
244
+ "entropy": 3.9387900829315186,
245
+ "epoch": 0.5416666666666666,
246
+ "frac_reward_zero_std": 0.25,
247
+ "grad_norm": 2.3026816844940186,
248
+ "kl": 0.08659639954566956,
249
+ "learning_rate": 4.5e-06,
250
+ "loss": -0.0018373141065239906,
251
+ "num_tokens": 100652.0,
252
+ "reward": -0.0949999988079071,
253
+ "reward_std": 0.14883993566036224,
254
+ "rewards/GeneratorRewardFunction/mean": -0.0949999988079071,
255
+ "rewards/GeneratorRewardFunction/std": 0.14883995056152344,
256
+ "step": 13,
257
+ "step_time": 12.306939647000036
258
+ },
259
+ {
260
+ "clip_ratio/high_max": 0.001953125,
261
+ "clip_ratio/high_mean": 0.001953125,
262
+ "clip_ratio/low_mean": 0.0,
263
+ "clip_ratio/low_min": 0.0,
264
+ "clip_ratio/region_mean": 0.001953125,
265
+ "entropy": 3.980618715286255,
266
+ "epoch": 0.5833333333333334,
267
+ "grad_norm": 2.359058380126953,
268
+ "kl": 0.10443169623613358,
269
+ "learning_rate": 4.4583333333333336e-06,
270
+ "loss": 0.06309118121862411,
271
+ "step": 14,
272
+ "step_time": 0.2221947499997441
273
+ },
274
+ {
275
+ "clip_ratio/high_max": 0.02669270895421505,
276
+ "clip_ratio/high_mean": 0.02669270895421505,
277
+ "clip_ratio/low_mean": 0.0,
278
+ "clip_ratio/low_min": 0.0,
279
+ "clip_ratio/region_mean": 0.02669270895421505,
280
+ "entropy": 3.6554672718048096,
281
+ "epoch": 0.625,
282
+ "grad_norm": 1.680372714996338,
283
+ "kl": 0.11937350034713745,
284
+ "learning_rate": 4.416666666666667e-06,
285
+ "loss": -0.11313736438751221,
286
+ "step": 15,
287
+ "step_time": 0.22094854400029362
288
+ },
289
+ {
290
+ "clip_ratio/high_max": 0.037109375,
291
+ "clip_ratio/high_mean": 0.037109375,
292
+ "clip_ratio/low_mean": 0.01692708395421505,
293
+ "clip_ratio/low_min": 0.01692708395421505,
294
+ "clip_ratio/region_mean": 0.0540364570915699,
295
+ "entropy": 3.9209201335906982,
296
+ "epoch": 0.6666666666666666,
297
+ "grad_norm": 2.50105619430542,
298
+ "kl": 0.10918638855218887,
299
+ "learning_rate": 4.3750000000000005e-06,
300
+ "loss": 0.0537506639957428,
301
+ "step": 16,
302
+ "step_time": 0.2213122180000937
303
+ },
304
+ {
305
+ "clip_ratio/high_max": 0.0,
306
+ "clip_ratio/high_mean": 0.0,
307
+ "clip_ratio/low_mean": 0.0,
308
+ "clip_ratio/low_min": 0.0,
309
+ "clip_ratio/region_mean": 0.0,
310
+ "completions/clipped_ratio": 1.0,
311
+ "completions/max_length": 384.0,
312
+ "completions/max_terminated_length": 0.0,
313
+ "completions/mean_length": 384.0,
314
+ "completions/mean_terminated_length": 0.0,
315
+ "completions/min_length": 384.0,
316
+ "completions/min_terminated_length": 0.0,
317
+ "entropy": 4.026664733886719,
318
+ "epoch": 0.7083333333333334,
319
+ "frac_reward_zero_std": 0.75,
320
+ "grad_norm": 0.3927060067653656,
321
+ "kl": 0.11360514163970947,
322
+ "learning_rate": 4.333333333333334e-06,
323
+ "loss": -0.015237320214509964,
324
+ "num_tokens": 125940.0,
325
+ "reward": -0.09406249970197678,
326
+ "reward_std": 0.1505458801984787,
327
+ "rewards/GeneratorRewardFunction/mean": -0.09406249970197678,
328
+ "rewards/GeneratorRewardFunction/std": 0.1505458652973175,
329
+ "step": 17,
330
+ "step_time": 12.339455734999774
331
+ },
332
+ {
333
+ "clip_ratio/high_max": 0.0,
334
+ "clip_ratio/high_mean": 0.0,
335
+ "clip_ratio/low_mean": 0.0,
336
+ "clip_ratio/low_min": 0.0,
337
+ "clip_ratio/region_mean": 0.0,
338
+ "entropy": 4.047895431518555,
339
+ "epoch": 0.75,
340
+ "grad_norm": 2.414126396179199,
341
+ "kl": 0.11749390512704849,
342
+ "learning_rate": 4.2916666666666665e-06,
343
+ "loss": 0.0929006040096283,
344
+ "step": 18,
345
+ "step_time": 0.230240794999645
346
+ },
347
+ {
348
+ "clip_ratio/high_max": 0.0032552082557231188,
349
+ "clip_ratio/high_mean": 0.0032552082557231188,
350
+ "clip_ratio/low_mean": 0.0,
351
+ "clip_ratio/low_min": 0.0,
352
+ "clip_ratio/region_mean": 0.0032552082557231188,
353
+ "entropy": 4.302440166473389,
354
+ "epoch": 0.7916666666666666,
355
+ "grad_norm": 1.0116335153579712,
356
+ "kl": 0.12855017185211182,
357
+ "learning_rate": 4.25e-06,
358
+ "loss": -0.03815174847841263,
359
+ "step": 19,
360
+ "step_time": 0.2287455929999851
361
+ },
362
+ {
363
+ "clip_ratio/high_max": 0.0078125,
364
+ "clip_ratio/high_mean": 0.0078125,
365
+ "clip_ratio/low_mean": 0.0,
366
+ "clip_ratio/low_min": 0.0,
367
+ "clip_ratio/region_mean": 0.0078125,
368
+ "entropy": 3.854883909225464,
369
+ "epoch": 0.8333333333333334,
370
+ "grad_norm": 0.9368992447853088,
371
+ "kl": 0.1246059313416481,
372
+ "learning_rate": 4.208333333333333e-06,
373
+ "loss": -0.03810252621769905,
374
+ "step": 20,
375
+ "step_time": 0.22873833199992077
376
+ },
377
+ {
378
+ "clip_ratio/high_max": 0.0,
379
+ "clip_ratio/high_mean": 0.0,
380
+ "clip_ratio/low_mean": 0.0,
381
+ "clip_ratio/low_min": 0.0,
382
+ "clip_ratio/region_mean": 0.0,
383
+ "completions/clipped_ratio": 1.0,
384
+ "completions/max_length": 384.0,
385
+ "completions/max_terminated_length": 0.0,
386
+ "completions/mean_length": 384.0,
387
+ "completions/mean_terminated_length": 0.0,
388
+ "completions/min_length": 384.0,
389
+ "completions/min_terminated_length": 0.0,
390
+ "entropy": 3.789822816848755,
391
+ "epoch": 0.875,
392
+ "frac_reward_zero_std": 0.5,
393
+ "grad_norm": 2.0554940700531006,
394
+ "kl": 0.15488088130950928,
395
+ "learning_rate": 4.166666666666667e-06,
396
+ "loss": -0.10783473402261734,
397
+ "num_tokens": 151364.0,
398
+ "reward": -0.16374999284744263,
399
+ "reward_std": 0.2437519133090973,
400
+ "rewards/GeneratorRewardFunction/mean": -0.16374999284744263,
401
+ "rewards/GeneratorRewardFunction/std": 0.24375192821025848,
402
+ "step": 21,
403
+ "step_time": 12.46333255899981
404
+ },
405
+ {
406
+ "clip_ratio/high_max": 0.0013020833721384406,
407
+ "clip_ratio/high_mean": 0.0013020833721384406,
408
+ "clip_ratio/low_mean": 0.0,
409
+ "clip_ratio/low_min": 0.0,
410
+ "clip_ratio/region_mean": 0.0013020833721384406,
411
+ "entropy": 4.022783279418945,
412
+ "epoch": 0.9166666666666666,
413
+ "grad_norm": 1.6437214612960815,
414
+ "kl": 0.14515824615955353,
415
+ "learning_rate": 4.125e-06,
416
+ "loss": -0.007372954394668341,
417
+ "step": 22,
418
+ "step_time": 0.22930750700015778
419
+ },
420
+ {
421
+ "clip_ratio/high_max": 0.0,
422
+ "clip_ratio/high_mean": 0.0,
423
+ "clip_ratio/low_mean": 0.001953125,
424
+ "clip_ratio/low_min": 0.001953125,
425
+ "clip_ratio/region_mean": 0.001953125,
426
+ "entropy": 4.210998058319092,
427
+ "epoch": 0.9583333333333334,
428
+ "grad_norm": 3.1202902793884277,
429
+ "kl": 0.15941737592220306,
430
+ "learning_rate": 4.083333333333334e-06,
431
+ "loss": 0.15042556822299957,
432
+ "step": 23,
433
+ "step_time": 0.22665810100033923
434
+ },
435
+ {
436
+ "clip_ratio/high_max": 0.01692708395421505,
437
+ "clip_ratio/high_mean": 0.01692708395421505,
438
+ "clip_ratio/low_mean": 0.0,
439
+ "clip_ratio/low_min": 0.0,
440
+ "clip_ratio/region_mean": 0.01692708395421505,
441
+ "entropy": 4.282694339752197,
442
+ "epoch": 1.0,
443
+ "grad_norm": 0.766208291053772,
444
+ "kl": 0.17048390209674835,
445
+ "learning_rate": 4.041666666666667e-06,
446
+ "loss": -0.03139518201351166,
447
+ "step": 24,
448
+ "step_time": 0.2268661449998035
449
+ },
450
+ {
451
+ "clip_ratio/high_max": 0.0,
452
+ "clip_ratio/high_mean": 0.0,
453
+ "clip_ratio/low_mean": 0.0,
454
+ "clip_ratio/low_min": 0.0,
455
+ "clip_ratio/region_mean": 0.0,
456
+ "completions/clipped_ratio": 1.0,
457
+ "completions/max_length": 384.0,
458
+ "completions/max_terminated_length": 0.0,
459
+ "completions/mean_length": 384.0,
460
+ "completions/mean_terminated_length": 0.0,
461
+ "completions/min_length": 384.0,
462
+ "completions/min_terminated_length": 0.0,
463
+ "entropy": 3.998028516769409,
464
+ "epoch": 1.0416666666666667,
465
+ "frac_reward_zero_std": 0.25,
466
+ "grad_norm": 2.7594497203826904,
467
+ "kl": 0.1617491990327835,
468
+ "learning_rate": 4.000000000000001e-06,
469
+ "loss": 0.03157578781247139,
470
+ "num_tokens": 176740.0,
471
+ "reward": -0.12992186844348907,
472
+ "reward_std": 0.20346397161483765,
473
+ "rewards/GeneratorRewardFunction/mean": -0.12992186844348907,
474
+ "rewards/GeneratorRewardFunction/std": 0.20346400141716003,
475
+ "step": 25,
476
+ "step_time": 12.400488669999959
477
+ },
478
+ {
479
+ "clip_ratio/high_max": 0.00390625,
480
+ "clip_ratio/high_mean": 0.00390625,
481
+ "clip_ratio/low_mean": 0.0,
482
+ "clip_ratio/low_min": 0.0,
483
+ "clip_ratio/region_mean": 0.00390625,
484
+ "entropy": 3.9673023223876953,
485
+ "epoch": 1.0833333333333333,
486
+ "grad_norm": 1.6026439666748047,
487
+ "kl": 0.1978183537721634,
488
+ "learning_rate": 3.958333333333333e-06,
489
+ "loss": -0.08369521051645279,
490
+ "step": 26,
491
+ "step_time": 0.22768656100015505
492
+ },
493
+ {
494
+ "clip_ratio/high_max": 0.00390625,
495
+ "clip_ratio/high_mean": 0.00390625,
496
+ "clip_ratio/low_mean": 0.0013020833721384406,
497
+ "clip_ratio/low_min": 0.0013020833721384406,
498
+ "clip_ratio/region_mean": 0.0052083334885537624,
499
+ "entropy": 4.1479878425598145,
500
+ "epoch": 1.125,
501
+ "grad_norm": 2.2012760639190674,
502
+ "kl": 0.16080449521541595,
503
+ "learning_rate": 3.916666666666667e-06,
504
+ "loss": 0.07530555874109268,
505
+ "step": 27,
506
+ "step_time": 0.22777760300004957
507
+ },
508
+ {
509
+ "clip_ratio/high_max": 0.01171875,
510
+ "clip_ratio/high_mean": 0.01171875,
511
+ "clip_ratio/low_mean": 0.0045572915114462376,
512
+ "clip_ratio/low_min": 0.0045572915114462376,
513
+ "clip_ratio/region_mean": 0.01627604104578495,
514
+ "entropy": 4.05542516708374,
515
+ "epoch": 1.1666666666666667,
516
+ "grad_norm": 2.911231517791748,
517
+ "kl": 0.165995791554451,
518
+ "learning_rate": 3.875e-06,
519
+ "loss": -0.02108343504369259,
520
+ "step": 28,
521
+ "step_time": 0.22810307000008834
522
+ },
523
+ {
524
+ "clip_ratio/high_max": 0.0,
525
+ "clip_ratio/high_mean": 0.0,
526
+ "clip_ratio/low_mean": 0.0,
527
+ "clip_ratio/low_min": 0.0,
528
+ "clip_ratio/region_mean": 0.0,
529
+ "completions/clipped_ratio": 1.0,
530
+ "completions/max_length": 384.0,
531
+ "completions/max_terminated_length": 0.0,
532
+ "completions/mean_length": 384.0,
533
+ "completions/mean_terminated_length": 0.0,
534
+ "completions/min_length": 384.0,
535
+ "completions/min_terminated_length": 0.0,
536
+ "entropy": 4.153504848480225,
537
+ "epoch": 1.2083333333333333,
538
+ "frac_reward_zero_std": 0.0,
539
+ "grad_norm": 2.6201553344726562,
540
+ "kl": 0.18019072711467743,
541
+ "learning_rate": 3.833333333333334e-06,
542
+ "loss": -0.002964133396744728,
543
+ "num_tokens": 201784.0,
544
+ "reward": -0.09195312112569809,
545
+ "reward_std": 0.1492348313331604,
546
+ "rewards/GeneratorRewardFunction/mean": -0.09195312112569809,
547
+ "rewards/GeneratorRewardFunction/std": 0.1492348462343216,
548
+ "step": 29,
549
+ "step_time": 12.342429325000012
550
+ },
551
+ {
552
+ "clip_ratio/high_max": 0.001953125,
553
+ "clip_ratio/high_mean": 0.001953125,
554
+ "clip_ratio/low_mean": 0.0,
555
+ "clip_ratio/low_min": 0.0,
556
+ "clip_ratio/region_mean": 0.001953125,
557
+ "entropy": 4.359671115875244,
558
+ "epoch": 1.25,
559
+ "grad_norm": 2.6328587532043457,
560
+ "kl": 0.17763398587703705,
561
+ "learning_rate": 3.7916666666666666e-06,
562
+ "loss": 0.002159977564588189,
563
+ "step": 30,
564
+ "step_time": 0.22143138000001272
565
+ },
566
+ {
567
+ "clip_ratio/high_max": 0.012369791977107525,
568
+ "clip_ratio/high_mean": 0.012369791977107525,
569
+ "clip_ratio/low_mean": 0.0006510416860692203,
570
+ "clip_ratio/low_min": 0.0006510416860692203,
571
+ "clip_ratio/region_mean": 0.013020833022892475,
572
+ "entropy": 3.472069501876831,
573
+ "epoch": 1.2916666666666667,
574
+ "grad_norm": 2.4978268146514893,
575
+ "kl": 0.20419950783252716,
576
+ "learning_rate": 3.7500000000000005e-06,
577
+ "loss": 0.0025085085071623325,
578
+ "step": 31,
579
+ "step_time": 0.22049251199996434
580
+ },
581
+ {
582
+ "clip_ratio/high_max": 0.02408854104578495,
583
+ "clip_ratio/high_mean": 0.02408854104578495,
584
+ "clip_ratio/low_mean": 0.009114583022892475,
585
+ "clip_ratio/low_min": 0.009114583022892475,
586
+ "clip_ratio/region_mean": 0.033203125,
587
+ "entropy": 4.1305012702941895,
588
+ "epoch": 1.3333333333333333,
589
+ "grad_norm": 2.5379302501678467,
590
+ "kl": 0.18195195496082306,
591
+ "learning_rate": 3.708333333333334e-06,
592
+ "loss": -0.0002308812690898776,
593
+ "step": 32,
594
+ "step_time": 0.21905139999989842
595
+ },
596
+ {
597
+ "clip_ratio/high_max": 0.0006510416860692203,
598
+ "clip_ratio/high_mean": 0.0006510416860692203,
599
+ "clip_ratio/low_mean": 0.0,
600
+ "clip_ratio/low_min": 0.0,
601
+ "clip_ratio/region_mean": 0.0006510416860692203,
602
+ "completions/clipped_ratio": 1.0,
603
+ "completions/max_length": 384.0,
604
+ "completions/max_terminated_length": 0.0,
605
+ "completions/mean_length": 384.0,
606
+ "completions/mean_terminated_length": 0.0,
607
+ "completions/min_length": 384.0,
608
+ "completions/min_terminated_length": 0.0,
609
+ "entropy": 4.056675434112549,
610
+ "epoch": 1.375,
611
+ "frac_reward_zero_std": 0.0,
612
+ "grad_norm": 2.5889852046966553,
613
+ "kl": 0.2006690949201584,
614
+ "learning_rate": 3.6666666666666666e-06,
615
+ "loss": 0.0013989699073135853,
616
+ "num_tokens": 226904.0,
617
+ "reward": -0.16500000655651093,
618
+ "reward_std": 0.24072805047035217,
619
+ "rewards/GeneratorRewardFunction/mean": -0.16500000655651093,
620
+ "rewards/GeneratorRewardFunction/std": 0.24072805047035217,
621
+ "step": 33,
622
+ "step_time": 12.293340841000372
623
+ },
624
+ {
625
+ "clip_ratio/high_max": 0.001953125,
626
+ "clip_ratio/high_mean": 0.001953125,
627
+ "clip_ratio/low_mean": 0.0013020833721384406,
628
+ "clip_ratio/low_min": 0.0013020833721384406,
629
+ "clip_ratio/region_mean": 0.0032552082557231188,
630
+ "entropy": 4.171106815338135,
631
+ "epoch": 1.4166666666666667,
632
+ "grad_norm": 2.8330881595611572,
633
+ "kl": 0.1881921887397766,
634
+ "learning_rate": 3.625e-06,
635
+ "loss": -0.022811776027083397,
636
+ "step": 34,
637
+ "step_time": 0.2281311049996475
638
+ },
639
+ {
640
+ "clip_ratio/high_max": 0.0045572915114462376,
641
+ "clip_ratio/high_mean": 0.0045572915114462376,
642
+ "clip_ratio/low_mean": 0.0071614584885537624,
643
+ "clip_ratio/low_min": 0.0071614584885537624,
644
+ "clip_ratio/region_mean": 0.01171875,
645
+ "entropy": 3.919452667236328,
646
+ "epoch": 1.4583333333333333,
647
+ "grad_norm": 2.5238964557647705,
648
+ "kl": 0.20949266850948334,
649
+ "learning_rate": 3.5833333333333335e-06,
650
+ "loss": -0.0009576271404512227,
651
+ "step": 35,
652
+ "step_time": 0.22765815499997188
653
+ },
654
+ {
655
+ "clip_ratio/high_max": 0.01627604104578495,
656
+ "clip_ratio/high_mean": 0.01627604104578495,
657
+ "clip_ratio/low_mean": 0.014973958022892475,
658
+ "clip_ratio/low_min": 0.014973958022892475,
659
+ "clip_ratio/region_mean": 0.03125,
660
+ "entropy": 3.9224002361297607,
661
+ "epoch": 1.5,
662
+ "grad_norm": 2.411428451538086,
663
+ "kl": 0.20672006905078888,
664
+ "learning_rate": 3.5416666666666673e-06,
665
+ "loss": 0.0244422759860754,
666
+ "step": 36,
667
+ "step_time": 0.2277545279998776
668
+ },
669
+ {
670
+ "clip_ratio/high_max": 0.0,
671
+ "clip_ratio/high_mean": 0.0,
672
+ "clip_ratio/low_mean": 0.0,
673
+ "clip_ratio/low_min": 0.0,
674
+ "clip_ratio/region_mean": 0.0,
675
+ "completions/clipped_ratio": 1.0,
676
+ "completions/max_length": 384.0,
677
+ "completions/max_terminated_length": 0.0,
678
+ "completions/mean_length": 384.0,
679
+ "completions/mean_terminated_length": 0.0,
680
+ "completions/min_length": 384.0,
681
+ "completions/min_terminated_length": 0.0,
682
+ "entropy": 4.0299153327941895,
683
+ "epoch": 1.5416666666666665,
684
+ "frac_reward_zero_std": 0.5,
685
+ "grad_norm": 1.6669820547103882,
686
+ "kl": 0.22218942642211914,
687
+ "learning_rate": 3.5e-06,
688
+ "loss": 0.02389518916606903,
689
+ "num_tokens": 252216.0,
690
+ "reward": -0.12593749165534973,
691
+ "reward_std": 0.20460708439350128,
692
+ "rewards/GeneratorRewardFunction/mean": -0.12593749165534973,
693
+ "rewards/GeneratorRewardFunction/std": 0.20460709929466248,
694
+ "step": 37,
695
+ "step_time": 12.253504645000248
696
+ },
697
+ {
698
+ "clip_ratio/high_max": 0.0,
699
+ "clip_ratio/high_mean": 0.0,
700
+ "clip_ratio/low_mean": 0.0,
701
+ "clip_ratio/low_min": 0.0,
702
+ "clip_ratio/region_mean": 0.0,
703
+ "entropy": 3.7865638732910156,
704
+ "epoch": 1.5833333333333335,
705
+ "grad_norm": 2.7283737659454346,
706
+ "kl": 0.1691073179244995,
707
+ "learning_rate": 3.4583333333333334e-06,
708
+ "loss": 0.007410774473100901,
709
+ "step": 38,
710
+ "step_time": 0.22871550900026705
711
+ },
712
+ {
713
+ "clip_ratio/high_max": 0.0013020833721384406,
714
+ "clip_ratio/high_mean": 0.0013020833721384406,
715
+ "clip_ratio/low_mean": 0.0,
716
+ "clip_ratio/low_min": 0.0,
717
+ "clip_ratio/region_mean": 0.0013020833721384406,
718
+ "entropy": 3.8288822174072266,
719
+ "epoch": 1.625,
720
+ "grad_norm": 1.3213032484054565,
721
+ "kl": 0.213203564286232,
722
+ "learning_rate": 3.416666666666667e-06,
723
+ "loss": -0.05342317000031471,
724
+ "step": 39,
725
+ "step_time": 0.2291703780001626
726
+ },
727
+ {
728
+ "clip_ratio/high_max": 0.005859375,
729
+ "clip_ratio/high_mean": 0.005859375,
730
+ "clip_ratio/low_mean": 0.0026041667442768812,
731
+ "clip_ratio/low_min": 0.0026041667442768812,
732
+ "clip_ratio/region_mean": 0.008463541977107525,
733
+ "entropy": 4.139598846435547,
734
+ "epoch": 1.6666666666666665,
735
+ "grad_norm": 1.4323750734329224,
736
+ "kl": 0.1957942247390747,
737
+ "learning_rate": 3.3750000000000003e-06,
738
+ "loss": 0.02381931059062481,
739
+ "step": 40,
740
+ "step_time": 0.2291786789996877
741
+ },
742
+ {
743
+ "clip_ratio/high_max": 0.0,
744
+ "clip_ratio/high_mean": 0.0,
745
+ "clip_ratio/low_mean": 0.0,
746
+ "clip_ratio/low_min": 0.0,
747
+ "clip_ratio/region_mean": 0.0,
748
+ "completions/clipped_ratio": 1.0,
749
+ "completions/max_length": 384.0,
750
+ "completions/max_terminated_length": 0.0,
751
+ "completions/mean_length": 384.0,
752
+ "completions/mean_terminated_length": 0.0,
753
+ "completions/min_length": 384.0,
754
+ "completions/min_terminated_length": 0.0,
755
+ "entropy": 3.784708261489868,
756
+ "epoch": 1.7083333333333335,
757
+ "frac_reward_zero_std": 0.75,
758
+ "grad_norm": 0.8244509100914001,
759
+ "kl": 0.23491795361042023,
760
+ "learning_rate": 3.3333333333333333e-06,
761
+ "loss": -0.030485158786177635,
762
+ "num_tokens": 277488.0,
763
+ "reward": -0.05218750238418579,
764
+ "reward_std": 0.008750000968575478,
765
+ "rewards/GeneratorRewardFunction/mean": -0.05218750238418579,
766
+ "rewards/GeneratorRewardFunction/std": 0.008750000037252903,
767
+ "step": 41,
768
+ "step_time": 12.314917613000034
769
+ },
770
+ {
771
+ "clip_ratio/high_max": 0.0,
772
+ "clip_ratio/high_mean": 0.0,
773
+ "clip_ratio/low_mean": 0.0,
774
+ "clip_ratio/low_min": 0.0,
775
+ "clip_ratio/region_mean": 0.0,
776
+ "entropy": 3.5051069259643555,
777
+ "epoch": 1.75,
778
+ "grad_norm": 0.0330439992249012,
779
+ "kl": 0.17619585990905762,
780
+ "learning_rate": 3.2916666666666668e-06,
781
+ "loss": 0.0004404896462801844,
782
+ "step": 42,
783
+ "step_time": 0.22708293400000912
784
+ },
785
+ {
786
+ "clip_ratio/high_max": 0.0,
787
+ "clip_ratio/high_mean": 0.0,
788
+ "clip_ratio/low_mean": 0.0,
789
+ "clip_ratio/low_min": 0.0,
790
+ "clip_ratio/region_mean": 0.0,
791
+ "entropy": 3.869413375854492,
792
+ "epoch": 1.7916666666666665,
793
+ "grad_norm": 2.3391029834747314,
794
+ "kl": 0.2293056696653366,
795
+ "learning_rate": 3.2500000000000002e-06,
796
+ "loss": 0.06325627863407135,
797
+ "step": 43,
798
+ "step_time": 0.22726368699977684
799
+ },
800
+ {
801
+ "clip_ratio/high_max": 0.005859375,
802
+ "clip_ratio/high_mean": 0.005859375,
803
+ "clip_ratio/low_mean": 0.0,
804
+ "clip_ratio/low_min": 0.0,
805
+ "clip_ratio/region_mean": 0.005859375,
806
+ "entropy": 3.9680118560791016,
807
+ "epoch": 1.8333333333333335,
808
+ "grad_norm": 0.7495855689048767,
809
+ "kl": 0.2308521717786789,
810
+ "learning_rate": 3.2083333333333337e-06,
811
+ "loss": -0.030605120584368706,
812
+ "step": 44,
813
+ "step_time": 0.22768630500013387
814
+ },
815
+ {
816
+ "clip_ratio/high_max": 0.0,
817
+ "clip_ratio/high_mean": 0.0,
818
+ "clip_ratio/low_mean": 0.0,
819
+ "clip_ratio/low_min": 0.0,
820
+ "clip_ratio/region_mean": 0.0,
821
+ "completions/clipped_ratio": 1.0,
822
+ "completions/max_length": 384.0,
823
+ "completions/max_terminated_length": 0.0,
824
+ "completions/mean_length": 384.0,
825
+ "completions/mean_terminated_length": 0.0,
826
+ "completions/min_length": 384.0,
827
+ "completions/min_terminated_length": 0.0,
828
+ "entropy": 3.8831920623779297,
829
+ "epoch": 1.875,
830
+ "frac_reward_zero_std": 1.0,
831
+ "grad_norm": 0.03628304600715637,
832
+ "kl": 0.19177107512950897,
833
+ "learning_rate": 3.1666666666666667e-06,
834
+ "loss": 0.00047942763194441795,
835
+ "num_tokens": 302728.0,
836
+ "reward": -0.05000000074505806,
837
+ "reward_std": 0.0,
838
+ "rewards/GeneratorRewardFunction/mean": -0.05000000074505806,
839
+ "rewards/GeneratorRewardFunction/std": 0.0,
840
+ "step": 45,
841
+ "step_time": 12.292037981000249
842
+ },
843
+ {
844
+ "clip_ratio/high_max": 0.0,
845
+ "clip_ratio/high_mean": 0.0,
846
+ "clip_ratio/low_mean": 0.0,
847
+ "clip_ratio/low_min": 0.0,
848
+ "clip_ratio/region_mean": 0.0,
849
+ "entropy": 4.143296241760254,
850
+ "epoch": 1.9166666666666665,
851
+ "grad_norm": 0.06912147253751755,
852
+ "kl": 0.22539162635803223,
853
+ "learning_rate": 3.125e-06,
854
+ "loss": 0.0005634790286421776,
855
+ "step": 46,
856
+ "step_time": 0.2225087099996017
857
+ },
858
+ {
859
+ "clip_ratio/high_max": 0.0,
860
+ "clip_ratio/high_mean": 0.0,
861
+ "clip_ratio/low_mean": 0.0,
862
+ "clip_ratio/low_min": 0.0,
863
+ "clip_ratio/region_mean": 0.0,
864
+ "entropy": 3.2712087631225586,
865
+ "epoch": 1.9583333333333335,
866
+ "grad_norm": 0.032519496977329254,
867
+ "kl": 0.17356155812740326,
868
+ "learning_rate": 3.0833333333333336e-06,
869
+ "loss": 0.0004339038860052824,
870
+ "step": 47,
871
+ "step_time": 0.22231021500010684
872
+ },
873
+ {
874
+ "clip_ratio/high_max": 0.0,
875
+ "clip_ratio/high_mean": 0.0,
876
+ "clip_ratio/low_mean": 0.0,
877
+ "clip_ratio/low_min": 0.0,
878
+ "clip_ratio/region_mean": 0.0,
879
+ "entropy": 3.385603189468384,
880
+ "epoch": 2.0,
881
+ "grad_norm": 0.037319280207157135,
882
+ "kl": 0.2101261019706726,
883
+ "learning_rate": 3.0416666666666666e-06,
884
+ "loss": 0.0005253152339719236,
885
+ "step": 48,
886
+ "step_time": 0.22219113299979654
887
+ },
888
+ {
889
+ "clip_ratio/high_max": 0.0,
890
+ "clip_ratio/high_mean": 0.0,
891
+ "clip_ratio/low_mean": 0.0,
892
+ "clip_ratio/low_min": 0.0,
893
+ "clip_ratio/region_mean": 0.0,
894
+ "completions/clipped_ratio": 1.0,
895
+ "completions/max_length": 384.0,
896
+ "completions/max_terminated_length": 0.0,
897
+ "completions/mean_length": 384.0,
898
+ "completions/mean_terminated_length": 0.0,
899
+ "completions/min_length": 384.0,
900
+ "completions/min_terminated_length": 0.0,
901
+ "entropy": 4.0906548500061035,
902
+ "epoch": 2.0416666666666665,
903
+ "frac_reward_zero_std": 0.5,
904
+ "grad_norm": 2.2955780029296875,
905
+ "kl": 0.19495069980621338,
906
+ "learning_rate": 3e-06,
907
+ "loss": 0.06160770729184151,
908
+ "num_tokens": 327732.0,
909
+ "reward": -0.052265629172325134,
910
+ "reward_std": 0.006680488586425781,
911
+ "rewards/GeneratorRewardFunction/mean": -0.052265629172325134,
912
+ "rewards/GeneratorRewardFunction/std": 0.006680487655103207,
913
+ "step": 49,
914
+ "step_time": 12.33933484399995
915
+ },
916
+ {
917
+ "clip_ratio/high_max": 0.0,
918
+ "clip_ratio/high_mean": 0.0,
919
+ "clip_ratio/low_mean": 0.0,
920
+ "clip_ratio/low_min": 0.0,
921
+ "clip_ratio/region_mean": 0.0,
922
+ "entropy": 3.150116205215454,
923
+ "epoch": 2.0833333333333335,
924
+ "grad_norm": 1.5421699285507202,
925
+ "kl": 0.16276103258132935,
926
+ "learning_rate": 2.9583333333333335e-06,
927
+ "loss": -0.09242680668830872,
928
+ "step": 50,
929
+ "step_time": 0.22293056900025476
930
+ },
931
+ {
932
+ "clip_ratio/high_max": 0.0026041667442768812,
933
+ "clip_ratio/high_mean": 0.0026041667442768812,
934
+ "clip_ratio/low_mean": 0.0,
935
+ "clip_ratio/low_min": 0.0,
936
+ "clip_ratio/region_mean": 0.0026041667442768812,
937
+ "entropy": 3.239654541015625,
938
+ "epoch": 2.125,
939
+ "grad_norm": 2.4545586109161377,
940
+ "kl": 0.22144116461277008,
941
+ "learning_rate": 2.916666666666667e-06,
942
+ "loss": 0.03188760578632355,
943
+ "step": 51,
944
+ "step_time": 0.22252764099994238
945
+ },
946
+ {
947
+ "clip_ratio/high_max": 0.0,
948
+ "clip_ratio/high_mean": 0.0,
949
+ "clip_ratio/low_mean": 0.0,
950
+ "clip_ratio/low_min": 0.0,
951
+ "clip_ratio/region_mean": 0.0,
952
+ "entropy": 3.705900192260742,
953
+ "epoch": 2.1666666666666665,
954
+ "grad_norm": 0.04030013456940651,
955
+ "kl": 0.20901231467723846,
956
+ "learning_rate": 2.875e-06,
957
+ "loss": 0.000522530812304467,
958
+ "step": 52,
959
+ "step_time": 0.2229615900000681
960
+ },
961
+ {
962
+ "clip_ratio/high_max": 0.0,
963
+ "clip_ratio/high_mean": 0.0,
964
+ "clip_ratio/low_mean": 0.0,
965
+ "clip_ratio/low_min": 0.0,
966
+ "clip_ratio/region_mean": 0.0,
967
+ "completions/clipped_ratio": 1.0,
968
+ "completions/max_length": 384.0,
969
+ "completions/max_terminated_length": 0.0,
970
+ "completions/mean_length": 384.0,
971
+ "completions/mean_terminated_length": 0.0,
972
+ "completions/min_length": 384.0,
973
+ "completions/min_terminated_length": 0.0,
974
+ "entropy": 3.891749620437622,
975
+ "epoch": 2.2083333333333335,
976
+ "frac_reward_zero_std": 0.75,
977
+ "grad_norm": 0.041814595460891724,
978
+ "kl": 0.2143976241350174,
979
+ "learning_rate": 2.8333333333333335e-06,
980
+ "loss": 0.0005359940696507692,
981
+ "num_tokens": 353224.0,
982
+ "reward": -0.05281250178813934,
983
+ "reward_std": 0.011249999515712261,
984
+ "rewards/GeneratorRewardFunction/mean": -0.05281250178813934,
985
+ "rewards/GeneratorRewardFunction/std": 0.011250000447034836,
986
+ "step": 53,
987
+ "step_time": 12.29338890400004
988
+ },
989
+ {
990
+ "clip_ratio/high_max": 0.0006510416860692203,
991
+ "clip_ratio/high_mean": 0.0006510416860692203,
992
+ "clip_ratio/low_mean": 0.0,
993
+ "clip_ratio/low_min": 0.0,
994
+ "clip_ratio/region_mean": 0.0006510416860692203,
995
+ "entropy": 3.85089373588562,
996
+ "epoch": 2.25,
997
+ "grad_norm": 2.5950942039489746,
998
+ "kl": 0.16594675183296204,
999
+ "learning_rate": 2.791666666666667e-06,
1000
+ "loss": 0.03165796399116516,
1001
+ "step": 54,
1002
+ "step_time": 0.22895524099976683
1003
+ },
1004
+ {
1005
+ "clip_ratio/high_max": 0.0006510416860692203,
1006
+ "clip_ratio/high_mean": 0.0006510416860692203,
1007
+ "clip_ratio/low_mean": 0.0,
1008
+ "clip_ratio/low_min": 0.0,
1009
+ "clip_ratio/region_mean": 0.0006510416860692203,
1010
+ "entropy": 3.740816354751587,
1011
+ "epoch": 2.2916666666666665,
1012
+ "grad_norm": 0.7538480758666992,
1013
+ "kl": 0.2336108237504959,
1014
+ "learning_rate": 2.7500000000000004e-06,
1015
+ "loss": -0.030538126826286316,
1016
+ "step": 55,
1017
+ "step_time": 0.22967811499984236
1018
+ },
1019
+ {
1020
+ "clip_ratio/high_max": 0.0,
1021
+ "clip_ratio/high_mean": 0.0,
1022
+ "clip_ratio/low_mean": 0.0,
1023
+ "clip_ratio/low_min": 0.0,
1024
+ "clip_ratio/region_mean": 0.0,
1025
+ "entropy": 3.5371694564819336,
1026
+ "epoch": 2.3333333333333335,
1027
+ "grad_norm": 0.03035014308989048,
1028
+ "kl": 0.1904689520597458,
1029
+ "learning_rate": 2.7083333333333334e-06,
1030
+ "loss": 0.00047617233940400183,
1031
+ "step": 56,
1032
+ "step_time": 0.22860865299981015
1033
+ },
1034
+ {
1035
+ "clip_ratio/high_max": 0.0,
1036
+ "clip_ratio/high_mean": 0.0,
1037
+ "clip_ratio/low_mean": 0.0,
1038
+ "clip_ratio/low_min": 0.0,
1039
+ "clip_ratio/region_mean": 0.0,
1040
+ "completions/clipped_ratio": 1.0,
1041
+ "completions/max_length": 384.0,
1042
+ "completions/max_terminated_length": 0.0,
1043
+ "completions/mean_length": 384.0,
1044
+ "completions/mean_terminated_length": 0.0,
1045
+ "completions/min_length": 384.0,
1046
+ "completions/min_terminated_length": 0.0,
1047
+ "entropy": 3.6744697093963623,
1048
+ "epoch": 2.375,
1049
+ "frac_reward_zero_std": 0.5,
1050
+ "grad_norm": 2.8526196479797363,
1051
+ "kl": 0.2179533690214157,
1052
+ "learning_rate": 2.666666666666667e-06,
1053
+ "loss": 0.06064089015126228,
1054
+ "num_tokens": 378200.0,
1055
+ "reward": -0.05062500387430191,
1056
+ "reward_std": 0.0017078256933018565,
1057
+ "rewards/GeneratorRewardFunction/mean": -0.05062500387430191,
1058
+ "rewards/GeneratorRewardFunction/std": 0.0017078248783946037,
1059
+ "step": 57,
1060
+ "step_time": 12.344568122000055
1061
+ },
1062
+ {
1063
+ "clip_ratio/high_max": 0.0,
1064
+ "clip_ratio/high_mean": 0.0,
1065
+ "clip_ratio/low_mean": 0.0,
1066
+ "clip_ratio/low_min": 0.0,
1067
+ "clip_ratio/region_mean": 0.0,
1068
+ "entropy": 3.5596373081207275,
1069
+ "epoch": 2.4166666666666665,
1070
+ "grad_norm": 2.336909532546997,
1071
+ "kl": 0.1864728480577469,
1072
+ "learning_rate": 2.6250000000000003e-06,
1073
+ "loss": 0.09075039625167847,
1074
+ "step": 58,
1075
+ "step_time": 0.22229691899974569
1076
+ },
1077
+ {
1078
+ "clip_ratio/high_max": 0.0013020833721384406,
1079
+ "clip_ratio/high_mean": 0.0013020833721384406,
1080
+ "clip_ratio/low_mean": 0.0,
1081
+ "clip_ratio/low_min": 0.0,
1082
+ "clip_ratio/region_mean": 0.0013020833721384406,
1083
+ "entropy": 4.0709099769592285,
1084
+ "epoch": 2.4583333333333335,
1085
+ "grad_norm": 1.0696587562561035,
1086
+ "kl": 0.19725312292575836,
1087
+ "learning_rate": 2.5833333333333337e-06,
1088
+ "loss": -0.05935385450720787,
1089
+ "step": 59,
1090
+ "step_time": 0.22132866900028603
1091
+ },
1092
+ {
1093
+ "clip_ratio/high_max": 0.005859375,
1094
+ "clip_ratio/high_mean": 0.005859375,
1095
+ "clip_ratio/low_mean": 0.0,
1096
+ "clip_ratio/low_min": 0.0,
1097
+ "clip_ratio/region_mean": 0.005859375,
1098
+ "entropy": 3.5235812664031982,
1099
+ "epoch": 2.5,
1100
+ "grad_norm": 1.339734435081482,
1101
+ "kl": 0.1914481371641159,
1102
+ "learning_rate": 2.5416666666666668e-06,
1103
+ "loss": -0.08955536037683487,
1104
+ "step": 60,
1105
+ "step_time": 0.2210670139998001
1106
+ },
1107
+ {
1108
+ "clip_ratio/high_max": 0.0,
1109
+ "clip_ratio/high_mean": 0.0,
1110
+ "clip_ratio/low_mean": 0.0,
1111
+ "clip_ratio/low_min": 0.0,
1112
+ "clip_ratio/region_mean": 0.0,
1113
+ "completions/clipped_ratio": 1.0,
1114
+ "completions/max_length": 384.0,
1115
+ "completions/max_terminated_length": 0.0,
1116
+ "completions/mean_length": 384.0,
1117
+ "completions/mean_terminated_length": 0.0,
1118
+ "completions/min_length": 384.0,
1119
+ "completions/min_terminated_length": 0.0,
1120
+ "entropy": 3.758277177810669,
1121
+ "epoch": 2.5416666666666665,
1122
+ "frac_reward_zero_std": 0.25,
1123
+ "grad_norm": 2.308443069458008,
1124
+ "kl": 0.20152616500854492,
1125
+ "learning_rate": 2.5e-06,
1126
+ "loss": 0.02940922975540161,
1127
+ "num_tokens": 403396.0,
1128
+ "reward": -0.09531249850988388,
1129
+ "reward_std": 0.15020506083965302,
1130
+ "rewards/GeneratorRewardFunction/mean": -0.09531249850988388,
1131
+ "rewards/GeneratorRewardFunction/std": 0.1502050757408142,
1132
+ "step": 61,
1133
+ "step_time": 12.279446078000092
1134
+ },
1135
+ {
1136
+ "clip_ratio/high_max": 0.0013020833721384406,
1137
+ "clip_ratio/high_mean": 0.0013020833721384406,
1138
+ "clip_ratio/low_mean": 0.0,
1139
+ "clip_ratio/low_min": 0.0,
1140
+ "clip_ratio/region_mean": 0.0013020833721384406,
1141
+ "entropy": 3.8132221698760986,
1142
+ "epoch": 2.5833333333333335,
1143
+ "grad_norm": 2.3874006271362305,
1144
+ "kl": 0.2049936056137085,
1145
+ "learning_rate": 2.4583333333333332e-06,
1146
+ "loss": 0.03161979839205742,
1147
+ "step": 62,
1148
+ "step_time": 0.2286197710000124
1149
+ },
1150
+ {
1151
+ "clip_ratio/high_max": 0.00390625,
1152
+ "clip_ratio/high_mean": 0.00390625,
1153
+ "clip_ratio/low_mean": 0.0,
1154
+ "clip_ratio/low_min": 0.0,
1155
+ "clip_ratio/region_mean": 0.00390625,
1156
+ "entropy": 3.9670188426971436,
1157
+ "epoch": 2.625,
1158
+ "grad_norm": 1.1636193990707397,
1159
+ "kl": 0.22340638935565948,
1160
+ "learning_rate": 2.4166666666666667e-06,
1161
+ "loss": -0.06298670917749405,
1162
+ "step": 63,
1163
+ "step_time": 0.22753021599964995
1164
+ },
1165
+ {
1166
+ "clip_ratio/high_max": 0.0078125,
1167
+ "clip_ratio/high_mean": 0.0078125,
1168
+ "clip_ratio/low_mean": 0.0013020833721384406,
1169
+ "clip_ratio/low_min": 0.0013020833721384406,
1170
+ "clip_ratio/region_mean": 0.009114583022892475,
1171
+ "entropy": 3.6722114086151123,
1172
+ "epoch": 2.6666666666666665,
1173
+ "grad_norm": 2.4340484142303467,
1174
+ "kl": 0.20960676670074463,
1175
+ "learning_rate": 2.375e-06,
1176
+ "loss": 0.00381827843375504,
1177
+ "step": 64,
1178
+ "step_time": 0.22782381399974838
1179
+ },
1180
+ {
1181
+ "clip_ratio/high_max": 0.0,
1182
+ "clip_ratio/high_mean": 0.0,
1183
+ "clip_ratio/low_mean": 0.0,
1184
+ "clip_ratio/low_min": 0.0,
1185
+ "clip_ratio/region_mean": 0.0,
1186
+ "completions/clipped_ratio": 1.0,
1187
+ "completions/max_length": 384.0,
1188
+ "completions/max_terminated_length": 0.0,
1189
+ "completions/mean_length": 384.0,
1190
+ "completions/mean_terminated_length": 0.0,
1191
+ "completions/min_length": 384.0,
1192
+ "completions/min_terminated_length": 0.0,
1193
+ "entropy": 3.7565574645996094,
1194
+ "epoch": 2.7083333333333335,
1195
+ "frac_reward_zero_std": 0.5,
1196
+ "grad_norm": 2.6330630779266357,
1197
+ "kl": 0.23081278800964355,
1198
+ "learning_rate": 2.3333333333333336e-06,
1199
+ "loss": 0.03251923248171806,
1200
+ "num_tokens": 428736.0,
1201
+ "reward": -0.08906249701976776,
1202
+ "reward_std": 0.14971326291561127,
1203
+ "rewards/GeneratorRewardFunction/mean": -0.08906249701976776,
1204
+ "rewards/GeneratorRewardFunction/std": 0.14971326291561127,
1205
+ "step": 65,
1206
+ "step_time": 12.274221834999935
1207
+ },
1208
+ {
1209
+ "clip_ratio/high_max": 0.001953125,
1210
+ "clip_ratio/high_mean": 0.001953125,
1211
+ "clip_ratio/low_mean": 0.0,
1212
+ "clip_ratio/low_min": 0.0,
1213
+ "clip_ratio/region_mean": 0.001953125,
1214
+ "entropy": 3.2141125202178955,
1215
+ "epoch": 2.75,
1216
+ "grad_norm": 0.7206648588180542,
1217
+ "kl": 0.20930321514606476,
1218
+ "learning_rate": 2.2916666666666666e-06,
1219
+ "loss": -0.030547266826033592,
1220
+ "step": 66,
1221
+ "step_time": 0.22652647299992168
1222
+ },
1223
+ {
1224
+ "clip_ratio/high_max": 0.0026041667442768812,
1225
+ "clip_ratio/high_mean": 0.0026041667442768812,
1226
+ "clip_ratio/low_mean": 0.0,
1227
+ "clip_ratio/low_min": 0.0,
1228
+ "clip_ratio/region_mean": 0.0026041667442768812,
1229
+ "entropy": 3.8311607837677,
1230
+ "epoch": 2.7916666666666665,
1231
+ "grad_norm": 1.0666906833648682,
1232
+ "kl": 0.20272450149059296,
1233
+ "learning_rate": 2.25e-06,
1234
+ "loss": -0.06231113150715828,
1235
+ "step": 67,
1236
+ "step_time": 0.22835564200022418
1237
+ },
1238
+ {
1239
+ "clip_ratio/high_max": 0.0032552082557231188,
1240
+ "clip_ratio/high_mean": 0.0032552082557231188,
1241
+ "clip_ratio/low_mean": 0.0013020833721384406,
1242
+ "clip_ratio/low_min": 0.0013020833721384406,
1243
+ "clip_ratio/region_mean": 0.0045572915114462376,
1244
+ "entropy": 3.2600362300872803,
1245
+ "epoch": 2.8333333333333335,
1246
+ "grad_norm": 2.548494815826416,
1247
+ "kl": 0.2023637741804123,
1248
+ "learning_rate": 2.2083333333333335e-06,
1249
+ "loss": 0.06257036328315735,
1250
+ "step": 68,
1251
+ "step_time": 0.22673189700026342
1252
+ },
1253
+ {
1254
+ "clip_ratio/high_max": 0.0,
1255
+ "clip_ratio/high_mean": 0.0,
1256
+ "clip_ratio/low_mean": 0.0,
1257
+ "clip_ratio/low_min": 0.0,
1258
+ "clip_ratio/region_mean": 0.0,
1259
+ "completions/clipped_ratio": 1.0,
1260
+ "completions/max_length": 384.0,
1261
+ "completions/max_terminated_length": 0.0,
1262
+ "completions/mean_length": 384.0,
1263
+ "completions/mean_terminated_length": 0.0,
1264
+ "completions/min_length": 384.0,
1265
+ "completions/min_terminated_length": 0.0,
1266
+ "entropy": 3.903346061706543,
1267
+ "epoch": 2.875,
1268
+ "frac_reward_zero_std": 0.75,
1269
+ "grad_norm": 2.5650641918182373,
1270
+ "kl": 0.20703309774398804,
1271
+ "learning_rate": 2.166666666666667e-06,
1272
+ "loss": 0.0005175751284696162,
1273
+ "num_tokens": 454092.0,
1274
+ "reward": -0.08749999850988388,
1275
+ "reward_std": 0.14999999105930328,
1276
+ "rewards/GeneratorRewardFunction/mean": -0.08749999850988388,
1277
+ "rewards/GeneratorRewardFunction/std": 0.15000000596046448,
1278
+ "step": 69,
1279
+ "step_time": 12.346748228000251
1280
+ },
1281
+ {
1282
+ "clip_ratio/high_max": 0.0,
1283
+ "clip_ratio/high_mean": 0.0,
1284
+ "clip_ratio/low_mean": 0.0,
1285
+ "clip_ratio/low_min": 0.0,
1286
+ "clip_ratio/region_mean": 0.0,
1287
+ "entropy": 3.604572296142578,
1288
+ "epoch": 2.9166666666666665,
1289
+ "grad_norm": 0.041312821209430695,
1290
+ "kl": 0.2584569454193115,
1291
+ "learning_rate": 2.125e-06,
1292
+ "loss": 0.0006461423472501338,
1293
+ "step": 70,
1294
+ "step_time": 0.22726081000018894
1295
+ },
1296
+ {
1297
+ "clip_ratio/high_max": 0.0,
1298
+ "clip_ratio/high_mean": 0.0,
1299
+ "clip_ratio/low_mean": 0.0,
1300
+ "clip_ratio/low_min": 0.0,
1301
+ "clip_ratio/region_mean": 0.0,
1302
+ "entropy": 3.936734437942505,
1303
+ "epoch": 2.9583333333333335,
1304
+ "grad_norm": 0.045058052986860275,
1305
+ "kl": 0.25572913885116577,
1306
+ "learning_rate": 2.0833333333333334e-06,
1307
+ "loss": 0.0006393228541128337,
1308
+ "step": 71,
1309
+ "step_time": 0.2266321950000929
1310
+ },
1311
+ {
1312
+ "clip_ratio/high_max": 0.0,
1313
+ "clip_ratio/high_mean": 0.0,
1314
+ "clip_ratio/low_mean": 0.0,
1315
+ "clip_ratio/low_min": 0.0,
1316
+ "clip_ratio/region_mean": 0.0,
1317
+ "entropy": 3.8187801837921143,
1318
+ "epoch": 3.0,
1319
+ "grad_norm": 0.15200896561145782,
1320
+ "kl": 0.2756902575492859,
1321
+ "learning_rate": 2.041666666666667e-06,
1322
+ "loss": 0.0006892255623824894,
1323
+ "step": 72,
1324
+ "step_time": 0.22748115400008828
1325
+ },
1326
+ {
1327
+ "clip_ratio/high_max": 0.0,
1328
+ "clip_ratio/high_mean": 0.0,
1329
+ "clip_ratio/low_mean": 0.0,
1330
+ "clip_ratio/low_min": 0.0,
1331
+ "clip_ratio/region_mean": 0.0,
1332
+ "completions/clipped_ratio": 1.0,
1333
+ "completions/max_length": 384.0,
1334
+ "completions/max_terminated_length": 0.0,
1335
+ "completions/mean_length": 384.0,
1336
+ "completions/mean_terminated_length": 0.0,
1337
+ "completions/min_length": 384.0,
1338
+ "completions/min_terminated_length": 0.0,
1339
+ "entropy": 3.5549018383026123,
1340
+ "epoch": 3.0416666666666665,
1341
+ "frac_reward_zero_std": 0.25,
1342
+ "grad_norm": 2.629650831222534,
1343
+ "kl": 0.27787163853645325,
1344
+ "learning_rate": 2.0000000000000003e-06,
1345
+ "loss": 0.0006946711800992489,
1346
+ "num_tokens": 479504.0,
1347
+ "reward": -0.1360156238079071,
1348
+ "reward_std": 0.20251502096652985,
1349
+ "rewards/GeneratorRewardFunction/mean": -0.1360156238079071,
1350
+ "rewards/GeneratorRewardFunction/std": 0.20251502096652985,
1351
+ "step": 73,
1352
+ "step_time": 12.36508035299994
1353
+ },
1354
+ {
1355
+ "clip_ratio/high_max": 0.0,
1356
+ "clip_ratio/high_mean": 0.0,
1357
+ "clip_ratio/low_mean": 0.0,
1358
+ "clip_ratio/low_min": 0.0,
1359
+ "clip_ratio/region_mean": 0.0,
1360
+ "entropy": 3.985502004623413,
1361
+ "epoch": 3.0833333333333335,
1362
+ "grad_norm": 3.8442986011505127,
1363
+ "kl": 0.23562563955783844,
1364
+ "learning_rate": 1.9583333333333334e-06,
1365
+ "loss": 0.16846270859241486,
1366
+ "step": 74,
1367
+ "step_time": 0.2228200750000724
1368
+ },
1369
+ {
1370
+ "clip_ratio/high_max": 0.0032552082557231188,
1371
+ "clip_ratio/high_mean": 0.0032552082557231188,
1372
+ "clip_ratio/low_mean": 0.0,
1373
+ "clip_ratio/low_min": 0.0,
1374
+ "clip_ratio/region_mean": 0.0032552082557231188,
1375
+ "entropy": 4.04638147354126,
1376
+ "epoch": 3.125,
1377
+ "grad_norm": 2.0279934406280518,
1378
+ "kl": 0.22598345577716827,
1379
+ "learning_rate": 1.916666666666667e-06,
1380
+ "loss": -0.12468577176332474,
1381
+ "step": 75,
1382
+ "step_time": 0.22250161700003446
1383
+ },
1384
+ {
1385
+ "clip_ratio/high_max": 0.0013020833721384406,
1386
+ "clip_ratio/high_mean": 0.0013020833721384406,
1387
+ "clip_ratio/low_mean": 0.0,
1388
+ "clip_ratio/low_min": 0.0,
1389
+ "clip_ratio/region_mean": 0.0013020833721384406,
1390
+ "entropy": 4.0013957023620605,
1391
+ "epoch": 3.1666666666666665,
1392
+ "grad_norm": 0.8672268986701965,
1393
+ "kl": 0.25679150223731995,
1394
+ "learning_rate": 1.8750000000000003e-06,
1395
+ "loss": -0.04179239273071289,
1396
+ "step": 76,
1397
+ "step_time": 0.22352542099997663
1398
+ },
1399
+ {
1400
+ "clip_ratio/high_max": 0.0,
1401
+ "clip_ratio/high_mean": 0.0,
1402
+ "clip_ratio/low_mean": 0.0,
1403
+ "clip_ratio/low_min": 0.0,
1404
+ "clip_ratio/region_mean": 0.0,
1405
+ "completions/clipped_ratio": 1.0,
1406
+ "completions/max_length": 384.0,
1407
+ "completions/max_terminated_length": 0.0,
1408
+ "completions/mean_length": 384.0,
1409
+ "completions/mean_terminated_length": 0.0,
1410
+ "completions/min_length": 384.0,
1411
+ "completions/min_terminated_length": 0.0,
1412
+ "entropy": 3.9860870838165283,
1413
+ "epoch": 3.2083333333333335,
1414
+ "frac_reward_zero_std": 0.5,
1415
+ "grad_norm": 2.1960203647613525,
1416
+ "kl": 0.2691328227519989,
1417
+ "learning_rate": 1.8333333333333333e-06,
1418
+ "loss": 0.09400800615549088,
1419
+ "num_tokens": 504260.0,
1420
+ "reward": -0.0950000062584877,
1421
+ "reward_std": 0.15186618268489838,
1422
+ "rewards/GeneratorRewardFunction/mean": -0.0950000062584877,
1423
+ "rewards/GeneratorRewardFunction/std": 0.15186616778373718,
1424
+ "step": 77,
1425
+ "step_time": 12.375023839000278
1426
+ },
1427
+ {
1428
+ "clip_ratio/high_max": 0.0,
1429
+ "clip_ratio/high_mean": 0.0,
1430
+ "clip_ratio/low_mean": 0.0,
1431
+ "clip_ratio/low_min": 0.0,
1432
+ "clip_ratio/region_mean": 0.0,
1433
+ "entropy": 4.034550189971924,
1434
+ "epoch": 3.25,
1435
+ "grad_norm": 1.1200612783432007,
1436
+ "kl": 0.27057135105133057,
1437
+ "learning_rate": 1.7916666666666667e-06,
1438
+ "loss": -0.05243874713778496,
1439
+ "step": 78,
1440
+ "step_time": 0.22202567900012582
1441
+ },
1442
+ {
1443
+ "clip_ratio/high_max": 0.0013020833721384406,
1444
+ "clip_ratio/high_mean": 0.0013020833721384406,
1445
+ "clip_ratio/low_mean": 0.0013020833721384406,
1446
+ "clip_ratio/low_min": 0.0013020833721384406,
1447
+ "clip_ratio/region_mean": 0.0026041667442768812,
1448
+ "entropy": 3.985944986343384,
1449
+ "epoch": 3.2916666666666665,
1450
+ "grad_norm": 2.5229761600494385,
1451
+ "kl": 0.25317808985710144,
1452
+ "learning_rate": 1.75e-06,
1453
+ "loss": 0.06283067911863327,
1454
+ "step": 79,
1455
+ "step_time": 0.22140788399974554
1456
+ },
1457
+ {
1458
+ "clip_ratio/high_max": 0.00390625,
1459
+ "clip_ratio/high_mean": 0.00390625,
1460
+ "clip_ratio/low_mean": 0.0,
1461
+ "clip_ratio/low_min": 0.0,
1462
+ "clip_ratio/region_mean": 0.00390625,
1463
+ "entropy": 3.7703773975372314,
1464
+ "epoch": 3.3333333333333335,
1465
+ "grad_norm": 1.491092562675476,
1466
+ "kl": 0.25448134541511536,
1467
+ "learning_rate": 1.7083333333333334e-06,
1468
+ "loss": -0.1019255518913269,
1469
+ "step": 80,
1470
+ "step_time": 0.2212280210001154
1471
+ },
1472
+ {
1473
+ "clip_ratio/high_max": 0.0,
1474
+ "clip_ratio/high_mean": 0.0,
1475
+ "clip_ratio/low_mean": 0.0,
1476
+ "clip_ratio/low_min": 0.0,
1477
+ "clip_ratio/region_mean": 0.0,
1478
+ "completions/clipped_ratio": 1.0,
1479
+ "completions/max_length": 384.0,
1480
+ "completions/max_terminated_length": 0.0,
1481
+ "completions/mean_length": 384.0,
1482
+ "completions/mean_terminated_length": 0.0,
1483
+ "completions/min_length": 384.0,
1484
+ "completions/min_terminated_length": 0.0,
1485
+ "entropy": 3.537435293197632,
1486
+ "epoch": 3.375,
1487
+ "frac_reward_zero_std": 0.25,
1488
+ "grad_norm": 1.2077699899673462,
1489
+ "kl": 0.25410404801368713,
1490
+ "learning_rate": 1.6666666666666667e-06,
1491
+ "loss": -0.030591806396842003,
1492
+ "num_tokens": 529376.0,
1493
+ "reward": -0.05781250074505806,
1494
+ "reward_std": 0.026010815054178238,
1495
+ "rewards/GeneratorRewardFunction/mean": -0.05781250074505806,
1496
+ "rewards/GeneratorRewardFunction/std": 0.026010815054178238,
1497
+ "step": 81,
1498
+ "step_time": 12.261559072000182
1499
+ },
1500
+ {
1501
+ "clip_ratio/high_max": 0.0006510416860692203,
1502
+ "clip_ratio/high_mean": 0.0006510416860692203,
1503
+ "clip_ratio/low_mean": 0.0,
1504
+ "clip_ratio/low_min": 0.0,
1505
+ "clip_ratio/region_mean": 0.0006510416860692203,
1506
+ "entropy": 3.698589563369751,
1507
+ "epoch": 3.4166666666666665,
1508
+ "grad_norm": 3.091418981552124,
1509
+ "kl": 0.23452502489089966,
1510
+ "learning_rate": 1.6250000000000001e-06,
1511
+ "loss": -0.05749604105949402,
1512
+ "step": 82,
1513
+ "step_time": 0.22881464900001447
1514
+ },
1515
+ {
1516
+ "clip_ratio/high_max": 0.0,
1517
+ "clip_ratio/high_mean": 0.0,
1518
+ "clip_ratio/low_mean": 0.0013020833721384406,
1519
+ "clip_ratio/low_min": 0.0013020833721384406,
1520
+ "clip_ratio/region_mean": 0.0013020833721384406,
1521
+ "entropy": 4.045624256134033,
1522
+ "epoch": 3.4583333333333335,
1523
+ "grad_norm": 0.7440784573554993,
1524
+ "kl": 0.2626417577266693,
1525
+ "learning_rate": 1.5833333333333333e-06,
1526
+ "loss": 0.030878538265824318,
1527
+ "step": 83,
1528
+ "step_time": 0.22878370899979927
1529
+ },
1530
+ {
1531
+ "clip_ratio/high_max": 0.0032552082557231188,
1532
+ "clip_ratio/high_mean": 0.0032552082557231188,
1533
+ "clip_ratio/low_mean": 0.0006510416860692203,
1534
+ "clip_ratio/low_min": 0.0006510416860692203,
1535
+ "clip_ratio/region_mean": 0.00390625,
1536
+ "entropy": 4.156180381774902,
1537
+ "epoch": 3.5,
1538
+ "grad_norm": 2.481998920440674,
1539
+ "kl": 0.24936126172542572,
1540
+ "learning_rate": 1.5416666666666668e-06,
1541
+ "loss": 0.060216207057237625,
1542
+ "step": 84,
1543
+ "step_time": 0.22899822399995173
1544
+ },
1545
+ {
1546
+ "clip_ratio/high_max": 0.0,
1547
+ "clip_ratio/high_mean": 0.0,
1548
+ "clip_ratio/low_mean": 0.0,
1549
+ "clip_ratio/low_min": 0.0,
1550
+ "clip_ratio/region_mean": 0.0,
1551
+ "completions/clipped_ratio": 1.0,
1552
+ "completions/max_length": 384.0,
1553
+ "completions/max_terminated_length": 0.0,
1554
+ "completions/mean_length": 384.0,
1555
+ "completions/mean_terminated_length": 0.0,
1556
+ "completions/min_length": 384.0,
1557
+ "completions/min_terminated_length": 0.0,
1558
+ "entropy": 3.9783565998077393,
1559
+ "epoch": 3.5416666666666665,
1560
+ "frac_reward_zero_std": 0.25,
1561
+ "grad_norm": 2.600086212158203,
1562
+ "kl": 0.26161837577819824,
1563
+ "learning_rate": 1.5e-06,
1564
+ "loss": 0.06223990395665169,
1565
+ "num_tokens": 554632.0,
1566
+ "reward": -0.06031249836087227,
1567
+ "reward_std": 0.025785572826862335,
1568
+ "rewards/GeneratorRewardFunction/mean": -0.06031249836087227,
1569
+ "rewards/GeneratorRewardFunction/std": 0.025785574689507484,
1570
+ "step": 85,
1571
+ "step_time": 12.248341717999665
1572
+ },
1573
+ {
1574
+ "clip_ratio/high_max": 0.0006510416860692203,
1575
+ "clip_ratio/high_mean": 0.0006510416860692203,
1576
+ "clip_ratio/low_mean": 0.0,
1577
+ "clip_ratio/low_min": 0.0,
1578
+ "clip_ratio/region_mean": 0.0006510416860692203,
1579
+ "entropy": 3.8679306507110596,
1580
+ "epoch": 3.5833333333333335,
1581
+ "grad_norm": 2.4648211002349854,
1582
+ "kl": 0.239444300532341,
1583
+ "learning_rate": 1.4583333333333335e-06,
1584
+ "loss": 0.0001831113622756675,
1585
+ "step": 86,
1586
+ "step_time": 0.22930015900010403
1587
+ },
1588
+ {
1589
+ "clip_ratio/high_max": 0.0,
1590
+ "clip_ratio/high_mean": 0.0,
1591
+ "clip_ratio/low_mean": 0.0,
1592
+ "clip_ratio/low_min": 0.0,
1593
+ "clip_ratio/region_mean": 0.0,
1594
+ "entropy": 3.973841428756714,
1595
+ "epoch": 3.625,
1596
+ "grad_norm": 2.234323024749756,
1597
+ "kl": 0.22450731694698334,
1598
+ "learning_rate": 1.4166666666666667e-06,
1599
+ "loss": 0.03148835897445679,
1600
+ "step": 87,
1601
+ "step_time": 0.22805973300000915
1602
+ },
1603
+ {
1604
+ "clip_ratio/high_max": 0.0013020833721384406,
1605
+ "clip_ratio/high_mean": 0.0013020833721384406,
1606
+ "clip_ratio/low_mean": 0.0,
1607
+ "clip_ratio/low_min": 0.0,
1608
+ "clip_ratio/region_mean": 0.0013020833721384406,
1609
+ "entropy": 3.7529680728912354,
1610
+ "epoch": 3.6666666666666665,
1611
+ "grad_norm": 1.4799991846084595,
1612
+ "kl": 0.2708089351654053,
1613
+ "learning_rate": 1.3750000000000002e-06,
1614
+ "loss": -0.09237971156835556,
1615
+ "step": 88,
1616
+ "step_time": 0.22939230100018904
1617
+ },
1618
+ {
1619
+ "clip_ratio/high_max": 0.0,
1620
+ "clip_ratio/high_mean": 0.0,
1621
+ "clip_ratio/low_mean": 0.0,
1622
+ "clip_ratio/low_min": 0.0,
1623
+ "clip_ratio/region_mean": 0.0,
1624
+ "completions/clipped_ratio": 1.0,
1625
+ "completions/max_length": 384.0,
1626
+ "completions/max_terminated_length": 0.0,
1627
+ "completions/mean_length": 384.0,
1628
+ "completions/mean_terminated_length": 0.0,
1629
+ "completions/min_length": 384.0,
1630
+ "completions/min_terminated_length": 0.0,
1631
+ "entropy": 3.905219793319702,
1632
+ "epoch": 3.7083333333333335,
1633
+ "frac_reward_zero_std": 0.25,
1634
+ "grad_norm": 2.375077486038208,
1635
+ "kl": 0.289053738117218,
1636
+ "learning_rate": 1.3333333333333334e-06,
1637
+ "loss": 0.06314738094806671,
1638
+ "num_tokens": 580076.0,
1639
+ "reward": -0.07375000417232513,
1640
+ "reward_std": 0.07433034479618073,
1641
+ "rewards/GeneratorRewardFunction/mean": -0.07375000417232513,
1642
+ "rewards/GeneratorRewardFunction/std": 0.07433034479618073,
1643
+ "step": 89,
1644
+ "step_time": 12.319528281999737
1645
+ },
1646
+ {
1647
+ "clip_ratio/high_max": 0.0006510416860692203,
1648
+ "clip_ratio/high_mean": 0.0006510416860692203,
1649
+ "clip_ratio/low_mean": 0.0,
1650
+ "clip_ratio/low_min": 0.0,
1651
+ "clip_ratio/region_mean": 0.0006510416860692203,
1652
+ "entropy": 3.588836431503296,
1653
+ "epoch": 3.75,
1654
+ "grad_norm": 1.3827015161514282,
1655
+ "kl": 0.2554599940776825,
1656
+ "learning_rate": 1.2916666666666669e-06,
1657
+ "loss": -0.09247452020645142,
1658
+ "step": 90,
1659
+ "step_time": 0.22871411200003422
1660
+ }
1661
+ ],
1662
+ "logging_steps": 1,
1663
+ "max_steps": 120,
1664
+ "num_input_tokens_seen": 580076,
1665
+ "num_train_epochs": 5,
1666
+ "save_steps": 30,
1667
+ "stateful_callbacks": {
1668
+ "TrainerControl": {
1669
+ "args": {
1670
+ "should_epoch_stop": false,
1671
+ "should_evaluate": false,
1672
+ "should_log": false,
1673
+ "should_save": true,
1674
+ "should_training_stop": false
1675
+ },
1676
+ "attributes": {}
1677
+ }
1678
+ },
1679
+ "total_flos": 0.0,
1680
+ "train_batch_size": 4,
1681
+ "trial_name": null,
1682
+ "trial_params": null
1683
+ }
self_play_hf_l40s_full/round_003/generator_train/checkpoint-90/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b445c5f8352c8b734291a26b36c1586326f270fa6c7a73403793a5b13784d278
3
+ size 7249