checkpoint 3230 adaptive

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:728e0719bfc2b88b533e21bdb4f2454d9fd976656cdd246883e6a01c7411a720
+oid sha256:3a1b3a22004ffbd1e028eaf577427ab4c4c8a0c73990101d3663c2045e2d7cd3
 size 1192135096

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b45ae3f39d88743b1662eb6134b02ad750d8cfaf6d404377e8efc8e59b6194d0
+oid sha256:e1c32b2c58d29f175e9bd52ca51e3628722368aa8031bf15bc96ed20689d731a
 size 2384460363

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91e62442493ae4f2963ab198bb4e1be4db65ab5cdd6b158248d98219357cb2b0
+oid sha256:e5b517d1b8e2b0f837c8b00170b154961d4d989feba4326ac25583df7a55c57a
 size 14645

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4af0f0a49e234227ff12101e0294b33302da01a67e316f5465c9475b2201d4ef
+oid sha256:d948e73f1c14a6b55827418ea52a1af9ca9e02d099d84d8e825e46a843f0ab61
 size 1465

trainer_state.json

@@ -1,996 +1,308 @@
 {
-  "best_global_step": 500,
-  "best_metric": 279.8433532714844,
-  "best_model_checkpoint": "output-dpo-contrastive-new/checkpoint-500",
-  "epoch": 3.0,
-  "eval_steps": 500,
-  "global_step": 9690,
+  "best_global_step": 800,
+  "best_metric": 209.9661102294922,
+  "best_model_checkpoint": "final-model-dpo-ad-1ep/checkpoint-800",
+  "epoch": 1.0,
+  "eval_steps": 800,
+  "global_step": 3230,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.03096634345545685,
-      "grad_norm": 1144.0,
-      "learning_rate": 9.997424714962634e-06,
-      "loss": 478.5263,
+      "grad_norm": 4.53125,
+      "learning_rate": 9.976838348954221e-06,
+      "loss": 5.5573,
       "step": 100
     },
     {
       "epoch": 0.0619326869109137,
-      "grad_norm": 249.0,
-      "learning_rate": 9.989597261051275e-06,
-      "loss": 5.8995,
+      "grad_norm": 4.15625,
+      "learning_rate": 9.906634890087323e-06,
+      "loss": 0.9191,
       "step": 200
     },
     {
       "epoch": 0.09289903036637055,
-      "grad_norm": 1416.0,
-      "learning_rate": 9.976525602356642e-06,
-      "loss": 1.974,
+      "grad_norm": 4.96875,
+      "learning_rate": 9.790050865156384e-06,
+      "loss": 1.0287,
       "step": 300
     },
     {
       "epoch": 0.1238653738218274,
-      "grad_norm": 124.0,
-      "learning_rate": 9.958223477553715e-06,
-      "loss": 1.2967,
+      "grad_norm": 11.625,
+      "learning_rate": 9.628188298907782e-06,
+      "loss": 1.0742,
       "step": 400
     },
     {
       "epoch": 0.15483171727728426,
-      "grad_norm": 185.0,
-      "learning_rate": 9.93471012268208e-06,
-      "loss": 0.9707,
-      "step": 500
-    },
-    {
-      "epoch": 0.15483171727728426,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -7700.236328125,
-      "eval_logps/rejected": -5658.39892578125,
-      "eval_loss": 279.8433532714844,
-      "eval_rewards/accuracies": 0.3200867176055908,
-      "eval_rewards/chosen": -724.652587890625,
-      "eval_rewards/margins": -192.2952880859375,
-      "eval_rewards/rejected": -532.3572998046875,
-      "eval_runtime": 1360.3414,
-      "eval_samples_per_second": 14.243,
-      "eval_steps_per_second": 14.243,
+      "grad_norm": 8.6875,
+      "learning_rate": 9.422577217034351e-06,
+      "loss": 1.2919,
       "step": 500
     },
     {
       "epoch": 0.1857980607327411,
-      "grad_norm": 136.0,
-      "learning_rate": 9.906010250928317e-06,
-      "loss": 0.8311,
+      "grad_norm": 16.75,
+      "learning_rate": 9.175161183420499e-06,
+      "loss": 0.9978,
       "step": 600
     },
     {
       "epoch": 0.21676440418819795,
-      "grad_norm": 66.5,
-      "learning_rate": 9.872154026651767e-06,
-      "loss": 0.6984,
+      "grad_norm": 5.1875,
+      "learning_rate": 8.888278928367003e-06,
+      "loss": 1.0586,
       "step": 700
     },
     {
       "epoch": 0.2477307476436548,
-      "grad_norm": 768.0,
-      "learning_rate": 9.833177033680945e-06,
-      "loss": 0.7479,
+      "grad_norm": 15.4375,
+      "learning_rate": 8.564642241456986e-06,
+      "loss": 1.4034,
       "step": 800
     },
     {
-      "epoch": 0.27869709109911167,
-      "grad_norm": 73.5,
-      "learning_rate": 9.789120237913954e-06,
-      "loss": 0.5771,
-      "step": 900
+      "epoch": 0.2477307476436548,
+      "eval_logits/chosen": NaN,
+      "eval_logits/rejected": NaN,
+      "eval_logps/chosen": -6010.39697265625,
+      "eval_logps/rejected": -4464.38623046875,
+      "eval_loss": 209.9661102294922,
+      "eval_rewards/accuracies": 0.3250929117202759,
+      "eval_rewards/chosen": -555.6686401367188,
+      "eval_rewards/margins": -142.71261596679688,
+      "eval_rewards/rejected": -412.95599365234375,
+      "eval_runtime": 1394.4098,
+      "eval_samples_per_second": 13.895,
+      "eval_steps_per_second": 13.895,
+      "step": 800
     },
     {
-      "epoch": 0.3096634345545685,
-      "grad_norm": 71.5,
-      "learning_rate": 9.740029944262193e-06,
-      "loss": 0.5536,
-      "step": 1000
+      "epoch": 0.27869709109911167,
+      "grad_norm": 13.875,
+      "learning_rate": 8.207310338033391e-06,
+      "loss": 1.5456,
+      "step": 900
     },
     {
       "epoch": 0.3096634345545685,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8134.36328125,
-      "eval_logps/rejected": -5972.11669921875,
-      "eval_loss": 297.4208679199219,
-      "eval_rewards/accuracies": 0.31869322061538696,
-      "eval_rewards/chosen": -768.0653686523438,
-      "eval_rewards/margins": -204.33627319335938,
-      "eval_rewards/rejected": -563.7291259765625,
-      "eval_runtime": 1355.184,
-      "eval_samples_per_second": 14.298,
-      "eval_steps_per_second": 14.298,
+      "grad_norm": 40.25,
+      "learning_rate": 7.819660941592014e-06,
+      "loss": 1.1894,
       "step": 1000
     },
     {
       "epoch": 0.34062977801002536,
-      "grad_norm": 53.5,
-      "learning_rate": 9.685957747982618e-06,
-      "loss": 0.5381,
+      "grad_norm": 40.25,
+      "learning_rate": 7.405358355437272e-06,
+      "loss": 1.4514,
       "step": 1100
     },
     {
       "epoch": 0.3715961214654822,
-      "grad_norm": 84.5,
-      "learning_rate": 9.626960480449692e-06,
-      "loss": 0.5969,
+      "grad_norm": 10.375,
+      "learning_rate": 6.968318825407323e-06,
+      "loss": 1.1933,
       "step": 1200
     },
     {
       "epoch": 0.40256246492093906,
-      "grad_norm": 70.5,
-      "learning_rate": 9.563100149424045e-06,
-      "loss": 0.4972,
+      "grad_norm": 28.125,
+      "learning_rate": 6.512673521081566e-06,
+      "loss": 1.3781,
       "step": 1300
     },
     {
       "epoch": 0.4335288083763959,
-      "grad_norm": 85.5,
-      "learning_rate": 9.4944438738806e-06,
-      "loss": 0.4799,
+      "grad_norm": 10.75,
+      "learning_rate": 6.042729485395221e-06,
+      "loss": 1.219,
       "step": 1400
     },
     {
       "epoch": 0.46449515183185275,
-      "grad_norm": 79.0,
-      "learning_rate": 9.421063813464661e-06,
-      "loss": 0.4689,
+      "grad_norm": 13.4375,
+      "learning_rate": 5.562928921789507e-06,
+      "loss": 1.3677,
       "step": 1500
     },
     {
-      "epoch": 0.46449515183185275,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8173.689453125,
-      "eval_logps/rejected": -5999.046875,
-      "eval_loss": 298.9976806640625,
-      "eval_rewards/accuracies": 0.3185900151729584,
-      "eval_rewards/chosen": -771.9979248046875,
-      "eval_rewards/margins": -205.575927734375,
-      "eval_rewards/rejected": -566.4219360351562,
-      "eval_runtime": 1358.7276,
-      "eval_samples_per_second": 14.26,
-      "eval_steps_per_second": 14.26,
-      "step": 1500
+      "epoch": 0.4954614952873096,
+      "grad_norm": 20.5,
+      "learning_rate": 5.077807203740619e-06,
+      "loss": 2.2353,
+      "step": 1600
     },
     {
       "epoch": 0.4954614952873096,
-      "grad_norm": 64.5,
-      "learning_rate": 9.343037092650156e-06,
-      "loss": 0.4951,
+      "eval_logits/chosen": NaN,
+      "eval_logits/rejected": NaN,
+      "eval_logps/chosen": -6157.478515625,
+      "eval_logps/rejected": -4588.53955078125,
+      "eval_loss": 214.6779022216797,
+      "eval_rewards/accuracies": 0.3266928195953369,
+      "eval_rewards/chosen": -570.3768310546875,
+      "eval_rewards/margins": -145.00547790527344,
+      "eval_rewards/rejected": -425.37127685546875,
+      "eval_runtime": 2581.7706,
+      "eval_samples_per_second": 7.505,
+      "eval_steps_per_second": 7.505,
       "step": 1600
     },
     {
       "epoch": 0.5264278387427664,
-      "grad_norm": 55.25,
-      "learning_rate": 9.260445719679651e-06,
-      "loss": 0.4569,
+      "grad_norm": 7.25,
+      "learning_rate": 4.591950003587562e-06,
+      "loss": 1.754,
       "step": 1700
     },
     {
       "epoch": 0.5573941821982233,
-      "grad_norm": 71.0,
-      "learning_rate": 9.173376500371441e-06,
-      "loss": 0.4647,
+      "grad_norm": 10.6875,
+      "learning_rate": 4.109949945903833e-06,
+      "loss": 1.6524,
       "step": 1800
     },
     {
       "epoch": 0.5883605256536801,
-      "grad_norm": 76.0,
-      "learning_rate": 9.081920946884217e-06,
-      "loss": 0.4954,
+      "grad_norm": 12.9375,
+      "learning_rate": 3.636363195152255e-06,
+      "loss": 1.2557,
       "step": 1900
     },
     {
       "epoch": 0.619326869109137,
-      "grad_norm": 48.75,
-      "learning_rate": 8.986175181535266e-06,
-      "loss": 0.4217,
-      "step": 2000
-    },
-    {
-      "epoch": 0.619326869109137,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8315.658203125,
-      "eval_logps/rejected": -6106.26806640625,
-      "eval_loss": 304.197021484375,
-      "eval_rewards/accuracies": 0.3195706009864807,
-      "eval_rewards/chosen": -786.1946411132812,
-      "eval_rewards/margins": -209.05052185058594,
-      "eval_rewards/rejected": -577.1441650390625,
-      "eval_runtime": 1355.5933,
-      "eval_samples_per_second": 14.293,
-      "eval_steps_per_second": 14.293,
+      "grad_norm": 27.5,
+      "learning_rate": 3.1756663879834735e-06,
+      "loss": 1.2763,
       "step": 2000
     },
     {
       "epoch": 0.6502932125645938,
-      "grad_norm": 43.75,
-      "learning_rate": 8.886239835773252e-06,
-      "loss": 0.4427,
+      "grad_norm": 18.25,
+      "learning_rate": 2.732214317280802e-06,
+      "loss": 1.7662,
       "step": 2100
     },
     {
       "epoch": 0.6812595560200507,
-      "grad_norm": 47.25,
-      "learning_rate": 8.782219944411774e-06,
-      "loss": 0.4756,
+      "grad_norm": 24.25,
+      "learning_rate": 2.3101987679481918e-06,
+      "loss": 1.7935,
       "step": 2200
     },
     {
       "epoch": 0.7122258994755075,
-      "grad_norm": 64.5,
-      "learning_rate": 8.674224835234879e-06,
-      "loss": 0.4246,
+      "grad_norm": 22.375,
+      "learning_rate": 1.913608893551036e-06,
+      "loss": 1.6459,
       "step": 2300
     },
     {
       "epoch": 0.7431922429309644,
-      "grad_norm": 71.5,
-      "learning_rate": 8.562368014090532e-06,
-      "loss": 0.4434,
+      "grad_norm": 32.75,
+      "learning_rate": 1.5461935083544755e-06,
+      "loss": 1.4079,
       "step": 2400
     },
     {
-      "epoch": 0.7741585863864212,
-      "grad_norm": 77.0,
-      "learning_rate": 8.446767045592829e-06,
-      "loss": 0.5978,
-      "step": 2500
+      "epoch": 0.7431922429309644,
+      "eval_logits/chosen": NaN,
+      "eval_logits/rejected": NaN,
+      "eval_logps/chosen": -6187.34716796875,
+      "eval_logps/rejected": -4600.21826171875,
+      "eval_loss": 216.61477661132812,
+      "eval_rewards/accuracies": 0.3254541754722595,
+      "eval_rewards/chosen": -573.3637084960938,
+      "eval_rewards/margins": -146.82444763183594,
+      "eval_rewards/rejected": -426.5391845703125,
+      "eval_runtime": 2856.8785,
+      "eval_samples_per_second": 6.782,
+      "eval_steps_per_second": 6.782,
+      "step": 2400
     },
     {
       "epoch": 0.7741585863864212,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8511.75,
-      "eval_logps/rejected": -6250.36181640625,
-      "eval_loss": 311.8685607910156,
-      "eval_rewards/accuracies": 0.3185900151729584,
-      "eval_rewards/chosen": -805.803955078125,
-      "eval_rewards/margins": -214.25033569335938,
-      "eval_rewards/rejected": -591.5535888671875,
-      "eval_runtime": 1359.1867,
-      "eval_samples_per_second": 14.256,
-      "eval_steps_per_second": 14.256,
+      "grad_norm": 16.375,
+      "learning_rate": 1.2114256511983274e-06,
+      "loss": 1.2724,
       "step": 2500
     },
     {
       "epoch": 0.8051249298418781,
-      "grad_norm": 49.75,
-      "learning_rate": 8.327543429558335e-06,
-      "loss": 0.4466,
+      "grad_norm": 12.6875,
+      "learning_rate": 9.124697561729073e-07,
+      "loss": 1.5263,
       "step": 2600
     },
     {
       "epoch": 0.836091273297335,
-      "grad_norm": 52.25,
-      "learning_rate": 8.20482247330641e-06,
-      "loss": 0.4372,
+      "grad_norm": 12.75,
+      "learning_rate": 6.521517404190009e-07,
+      "loss": 1.6869,
       "step": 2700
     },
     {
       "epoch": 0.8670576167527918,
-      "grad_norm": 62.0,
-      "learning_rate": 8.07873315995776e-06,
-      "loss": 0.4154,
+      "grad_norm": 11.625,
+      "learning_rate": 4.3293229180065233e-07,
+      "loss": 1.4792,
       "step": 2800
     },
     {
       "epoch": 0.8980239602082487,
-      "grad_norm": 69.0,
-      "learning_rate": 7.9494080128696e-06,
-      "loss": 0.4073,
+      "grad_norm": 9.0,
+      "learning_rate": 2.5688360895234796e-07,
+      "loss": 1.3264,
       "step": 2900
     },
     {
       "epoch": 0.9289903036637055,
-      "grad_norm": 46.75,
-      "learning_rate": 7.816982956349941e-06,
-      "loss": 0.399,
-      "step": 3000
-    },
-    {
-      "epoch": 0.9289903036637055,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8489.6416015625,
-      "eval_logps/rejected": -6233.3359375,
-      "eval_loss": 311.04168701171875,
-      "eval_rewards/accuracies": 0.31926095485687256,
-      "eval_rewards/chosen": -803.5930786132812,
-      "eval_rewards/margins": -213.7421875,
-      "eval_rewards/rejected": -589.8509521484375,
-      "eval_runtime": 1346.8135,
-      "eval_samples_per_second": 14.387,
-      "eval_steps_per_second": 14.387,
+      "grad_norm": 25.625,
+      "learning_rate": 1.256698135681289e-07,
+      "loss": 1.3937,
       "step": 3000
     },
     {
       "epoch": 0.9599566471191624,
-      "grad_norm": 40.75,
-      "learning_rate": 7.681597172797377e-06,
-      "loss": 0.4016,
+      "grad_norm": 21.25,
+      "learning_rate": 4.05312200878627e-08,
+      "loss": 1.6048,
       "step": 3100
     },
     {
       "epoch": 0.9909229905746192,
-      "grad_norm": 61.5,
-      "learning_rate": 7.543392956416542e-06,
-      "loss": 0.4017,
+      "grad_norm": 36.0,
+      "learning_rate": 2.272611473388975e-09,
+      "loss": 1.2954,
       "step": 3200
     },
     {
-      "epoch": 1.0216764404188199,
-      "grad_norm": 49.75,
-      "learning_rate": 7.4025155636629546e-06,
-      "loss": 0.3778,
-      "step": 3300
-    },
-    {
-      "epoch": 1.0526427838742767,
-      "grad_norm": 45.0,
-      "learning_rate": 7.25911306057447e-06,
-      "loss": 0.3591,
-      "step": 3400
-    },
-    {
-      "epoch": 1.0836091273297335,
-      "grad_norm": 61.5,
-      "learning_rate": 7.113336167149775e-06,
-      "loss": 0.3661,
-      "step": 3500
-    },
-    {
-      "epoch": 1.0836091273297335,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8532.7568359375,
-      "eval_logps/rejected": -6264.232421875,
-      "eval_loss": 312.83892822265625,
-      "eval_rewards/accuracies": 0.31828033924102783,
-      "eval_rewards/chosen": -807.90478515625,
-      "eval_rewards/margins": -214.964111328125,
-      "eval_rewards/rejected": -592.940673828125,
-      "eval_runtime": 1355.1706,
-      "eval_samples_per_second": 14.298,
-      "eval_steps_per_second": 14.298,
-      "step": 3500
-    },
-    {
-      "epoch": 1.1145754707851903,
-      "grad_norm": 41.25,
-      "learning_rate": 6.965338098937496e-06,
-      "loss": 0.3844,
-      "step": 3600
-    },
-    {
-      "epoch": 1.1455418142406473,
-      "grad_norm": 62.5,
-      "learning_rate": 6.815274406002428e-06,
-      "loss": 0.3653,
-      "step": 3700
-    },
-    {
-      "epoch": 1.176508157696104,
-      "grad_norm": 95.0,
-      "learning_rate": 6.663302809438097e-06,
-      "loss": 0.3915,
-      "step": 3800
-    },
-    {
-      "epoch": 1.2074745011515609,
-      "grad_norm": 50.25,
-      "learning_rate": 6.509583035597538e-06,
-      "loss": 0.3743,
-      "step": 3900
-    },
-    {
-      "epoch": 1.2384408446070179,
-      "grad_norm": 57.75,
-      "learning_rate": 6.35427664821648e-06,
-      "loss": 0.3638,
-      "step": 4000
-    },
-    {
-      "epoch": 1.2384408446070179,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8631.439453125,
-      "eval_logps/rejected": -6336.66796875,
-      "eval_loss": 316.6842346191406,
-      "eval_rewards/accuracies": 0.3185900151729584,
-      "eval_rewards/chosen": -817.7728271484375,
-      "eval_rewards/margins": -217.5886993408203,
-      "eval_rewards/rejected": -600.1841430664062,
-      "eval_runtime": 1356.2341,
-      "eval_samples_per_second": 14.287,
-      "eval_steps_per_second": 14.287,
-      "step": 4000
-    },
-    {
-      "epoch": 1.2694071880624747,
-      "grad_norm": 68.0,
-      "learning_rate": 6.197546878605376e-06,
-      "loss": 0.4077,
-      "step": 4100
-    },
-    {
-      "epoch": 1.3003735315179314,
-      "grad_norm": 44.0,
-      "learning_rate": 6.039558454088796e-06,
-      "loss": 0.3837,
-      "step": 4200
-    },
-    {
-      "epoch": 1.3313398749733882,
-      "grad_norm": 47.75,
-      "learning_rate": 5.88047742487244e-06,
-      "loss": 0.3916,
-      "step": 4300
-    },
-    {
-      "epoch": 1.362306218428845,
-      "grad_norm": 70.5,
-      "learning_rate": 5.720470989519773e-06,
-      "loss": 0.3604,
-      "step": 4400
-    },
-    {
-      "epoch": 1.393272561884302,
-      "grad_norm": 42.5,
-      "learning_rate": 5.559707319221725e-06,
-      "loss": 0.3803,
-      "step": 4500
-    },
-    {
-      "epoch": 1.393272561884302,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8557.1044921875,
-      "eval_logps/rejected": -6282.44775390625,
-      "eval_loss": 313.7301940917969,
-      "eval_rewards/accuracies": 0.3190028965473175,
-      "eval_rewards/chosen": -810.3394165039062,
-      "eval_rewards/margins": -215.5771942138672,
-      "eval_rewards/rejected": -594.7621459960938,
-      "eval_runtime": 1359.1293,
-      "eval_samples_per_second": 14.256,
-      "eval_steps_per_second": 14.256,
-      "step": 4500
-    },
-    {
-      "epoch": 1.4242389053397588,
-      "grad_norm": 37.25,
-      "learning_rate": 5.3983553810441004e-06,
-      "loss": 0.3853,
-      "step": 4600
-    },
-    {
-      "epoch": 1.4552052487952156,
-      "grad_norm": 49.0,
-      "learning_rate": 5.236584760338523e-06,
-      "loss": 0.3786,
-      "step": 4700
-    },
-    {
-      "epoch": 1.4861715922506726,
-      "grad_norm": 38.75,
-      "learning_rate": 5.074565482503543e-06,
-      "loss": 0.3855,
-      "step": 4800
-    },
-    {
-      "epoch": 1.5171379357061294,
-      "grad_norm": 36.5,
-      "learning_rate": 4.912467834283229e-06,
-      "loss": 0.3645,
-      "step": 4900
-    },
-    {
-      "epoch": 1.5481042791615862,
-      "grad_norm": 49.25,
-      "learning_rate": 4.750462184791093e-06,
-      "loss": 0.3757,
-      "step": 5000
-    },
-    {
-      "epoch": 1.5481042791615862,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8567.33203125,
-      "eval_logps/rejected": -6289.53173828125,
-      "eval_loss": 314.1842346191406,
-      "eval_rewards/accuracies": 0.3201383054256439,
-      "eval_rewards/chosen": -811.3621215820312,
-      "eval_rewards/margins": -215.89157104492188,
-      "eval_rewards/rejected": -595.470458984375,
-      "eval_runtime": 1365.2519,
-      "eval_samples_per_second": 14.192,
-      "eval_steps_per_second": 14.192,
-      "step": 5000
-    },
-    {
-      "epoch": 1.579070622617043,
-      "grad_norm": 82.5,
-      "learning_rate": 4.588718806447441e-06,
-      "loss": 0.3887,
-      "step": 5100
-    },
-    {
-      "epoch": 1.6100369660724998,
-      "grad_norm": 46.0,
-      "learning_rate": 4.427407696018343e-06,
-      "loss": 0.3759,
-      "step": 5200
-    },
-    {
-      "epoch": 1.6410033095279568,
-      "grad_norm": 39.25,
-      "learning_rate": 4.266698395944332e-06,
-      "loss": 0.3836,
-      "step": 5300
-    },
-    {
-      "epoch": 1.6719696529834136,
-      "grad_norm": 42.5,
-      "learning_rate": 4.1067598161466e-06,
-      "loss": 0.3727,
-      "step": 5400
-    },
-    {
-      "epoch": 1.7029359964388706,
-      "grad_norm": 64.5,
-      "learning_rate": 3.947760056498012e-06,
-      "loss": 0.3917,
-      "step": 5500
-    },
-    {
-      "epoch": 1.7029359964388706,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8585.994140625,
-      "eval_logps/rejected": -6303.09765625,
-      "eval_loss": 314.93072509765625,
-      "eval_rewards/accuracies": 0.31920933723449707,
-      "eval_rewards/chosen": -813.2284545898438,
-      "eval_rewards/margins": -216.4012451171875,
-      "eval_rewards/rejected": -596.8271484375,
-      "eval_runtime": 1366.4753,
-      "eval_samples_per_second": 14.18,
-      "eval_steps_per_second": 14.18,
-      "step": 5500
-    },
-    {
-      "epoch": 1.7339023398943274,
-      "grad_norm": 52.5,
-      "learning_rate": 3.7898662301454724e-06,
-      "loss": 0.3941,
-      "step": 5600
-    },
-    {
-      "epoch": 1.7648686833497842,
-      "grad_norm": 42.75,
-      "learning_rate": 3.6332442878693896e-06,
-      "loss": 0.3701,
-      "step": 5700
-    },
-    {
-      "epoch": 1.795835026805241,
-      "grad_norm": 42.75,
-      "learning_rate": 3.4780588436648223e-06,
-      "loss": 0.3707,
-      "step": 5800
-    },
-    {
-      "epoch": 1.8268013702606978,
-      "grad_norm": 52.75,
-      "learning_rate": 3.3244730017275974e-06,
-      "loss": 0.3729,
-      "step": 5900
-    },
-    {
-      "epoch": 1.8577677137161548,
-      "grad_norm": 49.75,
-      "learning_rate": 3.172648185027306e-06,
-      "loss": 0.3716,
-      "step": 6000
-    },
-    {
-      "epoch": 1.8577677137161548,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8575.375,
-      "eval_logps/rejected": -6295.24853515625,
-      "eval_loss": 314.5245361328125,
-      "eval_rewards/accuracies": 0.3183319568634033,
-      "eval_rewards/chosen": -812.16650390625,
-      "eval_rewards/margins": -216.12428283691406,
-      "eval_rewards/rejected": -596.042236328125,
-      "eval_runtime": 1368.3845,
-      "eval_samples_per_second": 14.16,
-      "eval_steps_per_second": 14.16,
-      "step": 6000
-    },
-    {
-      "epoch": 1.8887340571716116,
-      "grad_norm": 54.25,
-      "learning_rate": 3.0227439656472878e-06,
-      "loss": 0.3626,
-      "step": 6100
-    },
-    {
-      "epoch": 1.9197004006270686,
-      "grad_norm": 38.5,
-      "learning_rate": 2.87491789706995e-06,
-      "loss": 0.393,
-      "step": 6200
-    },
-    {
-      "epoch": 1.9506667440825254,
-      "grad_norm": 40.5,
-      "learning_rate": 2.729325348583711e-06,
-      "loss": 0.3619,
-      "step": 6300
-    },
-    {
-      "epoch": 1.9816330875379822,
-      "grad_norm": 80.0,
-      "learning_rate": 2.5861193419855634e-06,
-      "loss": 0.4156,
-      "step": 6400
-    },
-    {
-      "epoch": 2.0123865373821825,
-      "grad_norm": 82.5,
-      "learning_rate": 2.4454503907509493e-06,
-      "loss": 0.3607,
-      "step": 6500
-    },
-    {
-      "epoch": 2.0123865373821825,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8613.890625,
-      "eval_logps/rejected": -6323.0517578125,
-      "eval_loss": 316.09857177734375,
-      "eval_rewards/accuracies": 0.3194673955440521,
-      "eval_rewards/chosen": -816.01806640625,
-      "eval_rewards/margins": -217.19549560546875,
-      "eval_rewards/rejected": -598.8226318359375,
-      "eval_runtime": 1371.8671,
-      "eval_samples_per_second": 14.124,
-      "eval_steps_per_second": 14.124,
-      "step": 6500
-    },
-    {
-      "epoch": 2.0433528808376398,
-      "grad_norm": 43.25,
-      "learning_rate": 2.307466341839918e-06,
-      "loss": 0.3691,
-      "step": 6600
-    },
-    {
-      "epoch": 2.0743192242930966,
-      "grad_norm": 51.0,
-      "learning_rate": 2.1723122203058867e-06,
-      "loss": 0.3404,
-      "step": 6700
-    },
-    {
-      "epoch": 2.1052855677485534,
-      "grad_norm": 28.5,
-      "learning_rate": 2.040130076870296e-06,
-      "loss": 0.3692,
-      "step": 6800
-    },
-    {
-      "epoch": 2.13625191120401,
-      "grad_norm": 66.5,
-      "learning_rate": 1.9110588386233686e-06,
-      "loss": 0.3788,
-      "step": 6900
-    },
-    {
-      "epoch": 2.167218254659467,
-      "grad_norm": 83.0,
-      "learning_rate": 1.785234163007899e-06,
-      "loss": 0.3582,
-      "step": 7000
-    },
-    {
-      "epoch": 2.167218254659467,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8605.3466796875,
-      "eval_logps/rejected": -6316.7080078125,
-      "eval_loss": 315.78204345703125,
-      "eval_rewards/accuracies": 0.3188480734825134,
-      "eval_rewards/chosen": -815.16357421875,
-      "eval_rewards/margins": -216.9755096435547,
-      "eval_rewards/rejected": -598.1881103515625,
-      "eval_runtime": 1363.6925,
-      "eval_samples_per_second": 14.208,
-      "eval_steps_per_second": 14.208,
-      "step": 7000
-    },
-    {
-      "epoch": 2.1981845981149237,
-      "grad_norm": 49.5,
-      "learning_rate": 1.6627882952395197e-06,
-      "loss": 0.3762,
-      "step": 7100
-    },
-    {
-      "epoch": 2.2291509415703805,
-      "grad_norm": 43.25,
-      "learning_rate": 1.543849929313328e-06,
-      "loss": 0.368,
-      "step": 7200
-    },
-    {
-      "epoch": 2.2601172850258378,
-      "grad_norm": 43.0,
-      "learning_rate": 1.4285440727429296e-06,
-      "loss": 0.3496,
-      "step": 7300
-    },
-    {
-      "epoch": 2.2910836284812945,
-      "grad_norm": 53.0,
-      "learning_rate": 1.3169919151740884e-06,
-      "loss": 0.3826,
-      "step": 7400
-    },
-    {
-      "epoch": 2.3220499719367513,
-      "grad_norm": 42.5,
-      "learning_rate": 1.2093107010110516e-06,
-      "loss": 0.3704,
-      "step": 7500
-    },
-    {
-      "epoch": 2.3220499719367513,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8608.7568359375,
-      "eval_logps/rejected": -6319.107421875,
-      "eval_loss": 315.918701171875,
-      "eval_rewards/accuracies": 0.31920933723449707,
-      "eval_rewards/chosen": -815.5045166015625,
-      "eval_rewards/margins": -217.0764617919922,
-      "eval_rewards/rejected": -598.4281616210938,
-      "eval_runtime": 1364.6449,
-      "eval_samples_per_second": 14.199,
-      "eval_steps_per_second": 14.199,
-      "step": 7500
-    },
-    {
-      "epoch": 2.353016315392208,
-      "grad_norm": 52.75,
-      "learning_rate": 1.1056136061894386e-06,
-      "loss": 0.3592,
-      "step": 7600
-    },
-    {
-      "epoch": 2.383982658847665,
-      "grad_norm": 61.5,
-      "learning_rate": 1.006009619225199e-06,
-      "loss": 0.3523,
-      "step": 7700
-    },
-    {
-      "epoch": 2.4149490023031217,
-      "grad_norm": 40.75,
-      "learning_rate": 9.106034266646735e-07,
-      "loss": 0.3726,
-      "step": 7800
-    },
-    {
-      "epoch": 2.4459153457585785,
-      "grad_norm": 46.75,
-      "learning_rate": 8.194953030561226e-07,
-      "loss": 0.3816,
-      "step": 7900
-    },
-    {
-      "epoch": 2.4768816892140357,
-      "grad_norm": 78.0,
-      "learning_rate": 7.327810055584211e-07,
-      "loss": 0.3597,
-      "step": 8000
-    },
-    {
-      "epoch": 2.4768816892140357,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8607.6982421875,
-      "eval_logps/rejected": -6318.22314453125,
-      "eval_loss": 315.8908386230469,
-      "eval_rewards/accuracies": 0.31874483823776245,
-      "eval_rewards/chosen": -815.398681640625,
-      "eval_rewards/margins": -217.05902099609375,
-      "eval_rewards/rejected": -598.3396606445312,
-      "eval_runtime": 1361.5153,
-      "eval_samples_per_second": 14.231,
-      "eval_steps_per_second": 14.231,
-      "step": 8000
-    },
-    {
-      "epoch": 2.507848032669492,
-      "grad_norm": 46.75,
-      "learning_rate": 6.505516732976153e-07,
-      "loss": 0.3639,
-      "step": 8100
-    },
-    {
-      "epoch": 2.5388143761249493,
-      "grad_norm": 46.25,
-      "learning_rate": 5.728937315771954e-07,
-      "loss": 0.3778,
-      "step": 8200
-    },
-    {
-      "epoch": 2.569780719580406,
-      "grad_norm": 100.5,
-      "learning_rate": 4.99888801042701e-07,
-      "loss": 0.3645,
-      "step": 8300
-    },
-    {
-      "epoch": 2.600747063035863,
-      "grad_norm": 51.5,
-      "learning_rate": 4.316136118961656e-07,
-      "loss": 0.3746,
-      "step": 8400
-    },
-    {
-      "epoch": 2.6317134064913197,
-      "grad_norm": 67.5,
-      "learning_rate": 3.6813992325055504e-07,
-      "loss": 0.366,
-      "step": 8500
-    },
-    {
-      "epoch": 2.6317134064913197,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8610.2001953125,
-      "eval_logps/rejected": -6320.1904296875,
-      "eval_loss": 315.9734191894531,
-      "eval_rewards/accuracies": 0.3197254240512848,
-      "eval_rewards/chosen": -815.64892578125,
-      "eval_rewards/margins": -217.1125030517578,
-      "eval_rewards/rejected": -598.5364379882812,
-      "eval_runtime": 1480.2365,
-      "eval_samples_per_second": 13.09,
-      "eval_steps_per_second": 13.09,
-      "step": 8500
-    },
-    {
-      "epoch": 2.6626797499467765,
-      "grad_norm": 47.75,
-      "learning_rate": 3.095344477089462e-07,
-      "loss": 0.3551,
-      "step": 8600
-    },
-    {
-      "epoch": 2.6936460934022337,
-      "grad_norm": 54.75,
-      "learning_rate": 2.5585878124774754e-07,
-      "loss": 0.3719,
-      "step": 8700
-    },
-    {
-      "epoch": 2.72461243685769,
-      "grad_norm": 51.0,
-      "learning_rate": 2.0716933847761134e-07,
-      "loss": 0.3659,
-      "step": 8800
-    },
-    {
-      "epoch": 2.7555787803131473,
-      "grad_norm": 47.5,
-      "learning_rate": 1.6351729335012334e-07,
-      "loss": 0.3829,
-      "step": 8900
-    },
-    {
-      "epoch": 2.786545123768604,
-      "grad_norm": 63.5,
-      "learning_rate": 1.2494852537256296e-07,
-      "loss": 0.3503,
-      "step": 9000
-    },
-    {
-      "epoch": 2.786545123768604,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8609.8154296875,
-      "eval_logps/rejected": -6319.88720703125,
-      "eval_loss": 315.95770263671875,
-      "eval_rewards/accuracies": 0.3191577196121216,
-      "eval_rewards/chosen": -815.6104736328125,
-      "eval_rewards/margins": -217.10435485839844,
-      "eval_rewards/rejected": -598.5061645507812,
-      "eval_runtime": 1438.727,
-      "eval_samples_per_second": 13.467,
-      "eval_steps_per_second": 13.467,
-      "step": 9000
-    },
-    {
-      "epoch": 2.817511467224061,
-      "grad_norm": 46.5,
-      "learning_rate": 9.150357138727028e-08,
-      "loss": 0.3738,
-      "step": 9100
-    },
-    {
-      "epoch": 2.8484778106795177,
-      "grad_norm": 52.5,
-      "learning_rate": 6.321758296630398e-08,
-      "loss": 0.3754,
-      "step": 9200
-    },
-    {
-      "epoch": 2.8794441541349745,
-      "grad_norm": 68.5,
-      "learning_rate": 4.0120289466166754e-08,
-      "loss": 0.3747,
-      "step": 9300
-    },
-    {
-      "epoch": 2.9104104975904312,
-      "grad_norm": 46.75,
-      "learning_rate": 2.2235966781427586e-08,
-      "loss": 0.3565,
-      "step": 9400
-    },
-    {
-      "epoch": 2.941376841045888,
-      "grad_norm": 58.0,
-      "learning_rate": 9.583411830087485e-09,
-      "loss": 0.4153,
-      "step": 9500
-    },
-    {
-      "epoch": 2.941376841045888,
+      "epoch": 0.9909229905746192,
       "eval_logits/chosen": NaN,
       "eval_logits/rejected": NaN,
-      "eval_logps/chosen": -8609.7314453125,
-      "eval_logps/rejected": -6319.83056640625,
-      "eval_loss": 315.9565734863281,
-      "eval_rewards/accuracies": 0.3196222186088562,
-      "eval_rewards/chosen": -815.6021728515625,
-      "eval_rewards/margins": -217.1017303466797,
-      "eval_rewards/rejected": -598.50048828125,
-      "eval_runtime": 1365.7529,
-      "eval_samples_per_second": 14.187,
-      "eval_steps_per_second": 14.187,
-      "step": 9500
-    },
-    {
-      "epoch": 2.9723431845013453,
-      "grad_norm": 59.25,
-      "learning_rate": 2.1759227974949006e-09,
-      "loss": 0.3534,
-      "step": 9600
+      "eval_logps/chosen": -6186.20751953125,
+      "eval_logps/rejected": -4598.9970703125,
+      "eval_loss": 216.60252380371094,
+      "eval_rewards/accuracies": 0.3249380588531494,
+      "eval_rewards/chosen": -573.2498168945312,
+      "eval_rewards/margins": -146.83267211914062,
+      "eval_rewards/rejected": -426.41705322265625,
+      "eval_runtime": 1367.7925,
+      "eval_samples_per_second": 14.166,
+      "eval_steps_per_second": 14.166,
+      "step": 3200
     }
   ],
   "logging_steps": 100,
-  "max_steps": 9690,
+  "max_steps": 3230,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
-  "save_steps": 500,
+  "num_train_epochs": 1,
+  "save_steps": 800,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34b488da56bb162b28a10a08d84bea316241bb1080116bef2e50ef4a4c3ea7f4
-size 6609
+oid sha256:03c1fc65975d3429b9d2a5590beb102c89b663c463a8376f5d3653c19966cdbc
+size 6545