| Check QUESTION_TEMPLATE! |
|
|
| kl: 0.01 |
| lr: 1e-6 |
| epoch: 1 |
| with cold-start data |
|
|
| noise = torch.randn_like(advantages) * 0.02 |
| advantages = advantages + noise |
|
|
| def format_reward(completions, **kwargs): |
| pattern = r"<think>.*?</think>\s*<answer>.*?</answer>" |
| completion_contents = [completion[0]["content"] for completion in completions] |
| reward = [] |
| for content in completion_contents: |
| for_re = 0.0 |
| if re.fullmatch(pattern, content, re.DOTALL) and recheck_format(content): |
| for_re += 0.5 |
| think = extract_first_think_answer(content) |
| for_re += min(len(think) / 1200, 1) * 0.5 |
| reward.append(for_re) |
| return reward |
