Vikhrmodels/Llama-3-8B-Arena-General-RM-v4

RM Accuracy: 0.79551

Training config:

model_name_or_path: "princeton-nlp/gemma-2-9b-it-SimPO"
dataset: "Vikhrmodels/ru-arena-general-rankings"
per_device_train_batch_size: 4
per_device_eval_batch_size: 4
num_train_epochs: 3
save_strategy: "steps"
save_steps: 100
save_total_limit: 6
learning_rate: 0.0004
gradient_accumulation_steps: 4
attn_implementation: "flash_attention_2"
gradient_checkpointing: True
logging_steps: 1
remove_unused_columns: True
dataloader_num_workers: 2
max_length: 8192
center_rewards_coefficient: 0.01
test_size: 0.05
evaluation_strategy: "steps"
eval_steps: 50
run_name: "rm-arena-gemma-2-9b-it-simpo-lora-32-qkvougd-rc-0.01"
output_dir: "/mnt/models/rm-arena-gemma-2-9b-it-simpo-lora-32-qkvougd-0.01"
warmup_steps: 20
report_to: "wandb"
bf16: True
seed: 42
logging_first_step: True
use_peft: True
lora_task_type: SEQ_CLS
lora_target_modules:
  - "k_proj"
  - "v_proj"
  - "q_proj"
  - "o_proj"
  - "up_proj"
  - "gate_proj"
  - "down_proj"
lora_modules_to_save:
  - "score"
lora_r: 32
lora_alpha: 32
pad_token: "<pad>"
eos_token: "<eos>"
chat_template: "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] | trim + '\n\n' %}{% set messages = messages[1:] %}{% else %}{% set system_message = '' %}{% endif %}{% for message in messages %}{% if loop.index0 == 0 %}{% set content = system_message + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + content | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}"
force_chat_template: True